maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[flang] Replace Position with Provenance everywhere. 

Original-commit: flang-compiler/f18@8c2da3f8cc
Reviewed-on: https://github.com/flang-compiler/f18/pull/9
Tree-same-pre-rewrite: false
This commit is contained in:
peter klausler 2018-02-09 14:04:11 -08:00 committed by GitHub
parent 3558c22101
commit 09865ffe7b
24 changed files with 692 additions and 627 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
flang/CMakeLists.txt
View File

@ -5,8 +5,7 @@ set(CMAKE_CXX_COMPILER "${GCC}/bin/g++")
set(CMAKE_INSTALL_RPATH "${GCC}/lib64")
set(CMAKE_BUILD_WITH_INSTALL_RPATH true)
project(f18)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++17")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -pedantic -std=c++17")
set(SOURCES_F18
tools/f18/f18.cc
@ -14,7 +13,6 @@ set(SOURCES_F18
lib/parser/idioms.cc
lib/parser/message.cc
lib/parser/parse-tree.cc
lib/parser/position.cc
lib/parser/preprocessor.cc
lib/parser/prescan.cc
lib/parser/provenance.cc

49
flang/lib/parser/basic-parsers.h
View File

@ -17,7 +17,7 @@
#include "idioms.h"
#include "message.h"
#include "parse-state.h"
#include "position.h"
#include "provenance.h"
#include <cstring>
#include <functional>
#include <list>
@ -38,7 +38,7 @@ public:
constexpr FailParser(const FailParser &) = default;
constexpr explicit FailParser(const char *str) : str_{str} {}
std::optional<A> Parse(ParseState *state) const {
state->messages()->Add(Message{state->position(), str_, state->context()});
state->PutMessage(str_);
return {};
}
@ -252,7 +252,7 @@ public:
}
// Both alternatives failed. Retain the state (and messages) from the
// alternative parse that went the furthest.
if (state->position() <= paState.position()) {
if (state->GetLocation() <= paState.GetLocation()) {
messages.Annex(paState.messages());
state->swap(paState);
} else {
@ -332,13 +332,13 @@ public:
constexpr ManyParser(const PA &parser) : parser_{parser} {}
std::optional<resultType> Parse(ParseState *state) const {
resultType result;
Position at{state->position()};
auto at = state->GetLocation();
while (std::optional<paType> x{parser_.Parse(state)}) {
result.emplace_back(std::move(*x));
if (state->position() <= at) {
if (state->GetLocation() <= at) {
break; // no forward progress, don't loop
}
at = state->position();
at = state->GetLocation();
}
return {std::move(result)};
}
@ -363,11 +363,11 @@ public:
constexpr SomeParser(const SomeParser &) = default;
constexpr SomeParser(const PA &parser) : parser_{parser} {}
std::optional<resultType> Parse(ParseState *state) const {
Position start{state->position()};
auto start = state->GetLocation();
if (std::optional<paType> first{parser_.Parse(state)}) {
resultType result;
result.emplace_back(std::move(*first));
if ((state->position() > start)) {
if ((state->GetLocation() > start)) {
result.splice(result.end(), *many(parser_).Parse(state));
}
return {std::move(result)};
@ -390,9 +390,9 @@ public:
constexpr SkipManyParser(const SkipManyParser &) = default;
constexpr SkipManyParser(const PA &parser) : parser_{parser} {}
std::optional<Success> Parse(ParseState *state) const {
for (Position at{state->position()};
parser_.Parse(state) && state->position() > at;
at = state->position()) {
for (auto at = state->GetLocation();
parser_.Parse(state) && state->GetLocation() > at;
at = state->GetLocation()) {
}
return {Success{}};
}
@ -1190,8 +1190,7 @@ constexpr struct RawNextCharParser {
state->Advance();
return ch;
}
state->messages()->Add(
Message{state->position(), "end of file", state->context()});
state->PutMessage("end of file");
return {};
}
} rawNextChar;
@ -1232,12 +1231,11 @@ public:
if (state->strictConformance()) {
return {};
}
Position at{state->position()};
auto at = state->GetLocation();
auto result = parser_.Parse(state);
if (result) {
if (state->warnOnNonstandardUsage()) {
state->messages()->Add(
Message{at, "nonstandard usage", state->context()});
state->PutMessage(at, "nonstandard usage");
}
}
return result;
@ -1263,12 +1261,11 @@ public:
if (state->strictConformance()) {
return {};
}
Position at{state->position()};
auto at = state->GetLocation();
auto result = parser_.Parse(state);
if (result) {
if (state->warnOnDeprecatedUsage()) {
state->messages()->Add(
Message{at, "deprecated usage", state->context()});
state->PutMessage(at, "deprecated usage");
}
}
return result;
@ -1305,17 +1302,17 @@ constexpr struct GetColumn {
using resultType = int;
constexpr GetColumn() {}
static std::optional<int> Parse(ParseState *state) {
return {state->position().column()};
return {state->column()};
}
} getColumn;
constexpr struct GetPosition {
using resultType = Position;
constexpr GetPosition() {}
static std::optional<Position> Parse(ParseState *state) {
return {state->position()};
constexpr struct GetProvenance {
using resultType = Provenance;
constexpr GetProvenance() {}
static std::optional<Provenance> Parse(ParseState *state) {
return {state->GetProvenance()};
}
} getPosition;
} getProvenance;
} // namespace parser
} // namespace Fortran

8
flang/lib/parser/char-buffer.cc
View File

@ -38,13 +38,5 @@ void CharBuffer::Put(const char *data, size_t n) {
}
void CharBuffer::Put(const std::string &str) { Put(str.data(), str.size()); }
void CharBuffer::CopyToContiguous(char *data) {
char *to{data};
for (char ch : *this) {
*to++ = ch;
}
CHECK(to == data + bytes_);
}
} // namespace parser
} // namespace Fortran

3
flang/lib/parser/char-buffer.h
View File

@ -29,7 +29,7 @@ public:
return *this;
}
size_t bytes() const { return bytes_; }
size_t size() const { return bytes_; }
void clear() {
blocks_.clear();
@ -43,7 +43,6 @@ public:
void Put(const char *data, size_t n);
void Put(const std::string &);
void Put(char x) { Put(&x, 1); }
void CopyToContiguous(char *data);
private:
struct Block {

16
flang/lib/parser/cooked-chars.h
View File

@ -24,10 +24,10 @@ constexpr struct FixedFormPadding {
using resultType = char;
static std::optional<char> Parse(ParseState *state) {
if (state->inCharLiteral() && state->inFortran() && state->inFixedForm() &&
state->position().column() <= state->columns()) {
state->column() <= state->columns()) {
if (std::optional<char> ch{state->GetNextRawChar()}) {
if (*ch == '\n') {
state->AdvancePositionForPadding();
state->AdvanceColumnForPadding();
return {' '};
}
}
@ -44,8 +44,7 @@ constexpr StateUpdateParser noteSkippedNewLine{IncrementSkippedNewLines};
static inline bool InRightMargin(const ParseState &state) {
if (state.inFortran() && state.inFixedForm() &&
state.position().column() > state.columns() &&
!state.tabInCurrentLine()) {
state.column() > state.columns() && !state.tabInCurrentLine()) {
if (std::optional<char> ch{state.GetNextRawChar()}) {
return *ch != '\n';
}
@ -61,7 +60,7 @@ template<int col> struct AtFixedFormColumn {
constexpr AtFixedFormColumn(const AtFixedFormColumn &) {}
static std::optional<Success> Parse(ParseState *state) {
if (state->inFortran() && state->inFixedForm() && !state->IsAtEnd() &&
state->position().column() == col) {
state->column() == col) {
return {Success{}};
}
return {};
@ -73,7 +72,7 @@ template<int col> struct AtColumn {
constexpr AtColumn() {}
constexpr AtColumn(const AtColumn &) {}
static std::optional<Success> Parse(ParseState *state) {
if (!state->IsAtEnd() && state->position().column() == col) {
if (!state->IsAtEnd() && state->column() == col) {
return {Success{}};
}
return {};
@ -81,8 +80,7 @@ template<int col> struct AtColumn {
};
static inline bool AtOldDebugLineMarker(const ParseState &state) {
if (state.inFortran() && state.inFixedForm() &&
state.position().column() == 1) {
if (state.inFortran() && state.inFixedForm() && state.column() == 1) {
if (std::optional<char> ch{state.GetNextRawChar()}) {
return toupper(*ch) == 'D';
}
@ -114,7 +112,7 @@ constexpr struct FastRawSpaceParser {
static std::optional<Success> Parse(ParseState *state) {
if (std::optional<char> ch{state->GetNextRawChar()}) {
if (*ch == ' ' || *ch == '\t' ||
(toupper(*ch) == 'D' && state->position().column() == 1 &&
(toupper(*ch) == 'D' && state->column() == 1 &&
state->enableOldDebugLines() && state->inFortran() &&
state->inFixedForm())) {
state->Advance();

49
flang/lib/parser/cooked-tokens.h
View File

@ -8,7 +8,7 @@
#include "basic-parsers.h"
#include "cooked-chars.h"
#include "idioms.h"
#include "position.h"
#include "provenance.h"
#include <cctype>
#include <cstring>
#include <functional>
@ -28,13 +28,13 @@ public:
constexpr CharPredicateGuardParser(bool (*f)(char), const char *msg)
: predicate_{f}, message_{msg} {}
std::optional<char> Parse(ParseState *state) const {
Position at{state->position()};
auto at = state->GetLocation();
if (std::optional<char> result{cookedNextChar.Parse(state)}) {
if (predicate_(*result)) {
return result;
}
}
state->messages()->Add(Message{at, message_, state->context()});
state->PutMessage(at, message_);
return {};
}
@ -67,13 +67,13 @@ public:
using resultType = char;
constexpr CharMatch() {}
static std::optional<char> Parse(ParseState *state) {
Position at{state->position()};
auto at = state->GetLocation();
std::optional<char> result{cookedNextChar.Parse(state)};
if (result && *result != good) {
result.reset();
}
if (!result) {
state->messages()->Add(Message{at, good, state->context()});
state->PutMessage(at, "expected '"s + good + '\'');
}
return {result};
}
@ -103,7 +103,7 @@ public:
: str_{str}, length_{n} {}
constexpr TokenStringMatch(const char *str) : str_{str} {}
std::optional<Success> Parse(ParseState *state) const {
Position at{state->position()};
auto at = state->GetLocation();
if (!spaces.Parse(state)) {
return {};
}
@ -130,8 +130,7 @@ public:
} else if (*ch == tolower(*p)) {
ch.reset();
} else {
state->messages()->Add(
Message{at, "expected '"s + str_ + '\'', state->context()});
state->PutMessage(at, "expected '"s + str_ + '\'');
return {};
}
}
@ -191,15 +190,14 @@ struct CharLiteralChar {
};
using resultType = Result;
static std::optional<Result> Parse(ParseState *state) {
Position at{state->position()};
auto at = state->GetLocation();
std::optional<char> och{cookedNextChar.Parse(state)};
if (!och.has_value()) {
return {};
}
char ch{*och};
if (ch == '\n') {
state->messages()->Add(
Message{at, "unclosed character constant", state->context()});
state->PutMessage(at, "unclosed character constant");
return {};
}
if (ch != '\\' || !state->enableBackslashEscapesInCharLiterals()) {
@ -219,10 +217,7 @@ struct CharLiteralChar {
case '"':
case '\'':
case '\\': return {Result::Escaped(ch)};
case '\n':
state->messages()->Add(
Message{at, "unclosed character constant", state->context()});
return {};
case '\n': state->PutMessage(at, "unclosed character constant"); return {};
default:
if (IsOctalDigit(ch)) {
ch -= '0';
@ -243,8 +238,7 @@ struct CharLiteralChar {
}
}
} else {
state->messages()->Add(
Message{at, "bad escaped character", state->context()});
state->PutMessage(at, "bad escaped character");
}
return {Result::Escaped(ch)};
}
@ -308,7 +302,7 @@ struct BOZLiteral {
return {};
}
Position at{state->position()};
auto at = state->GetLocation();
std::string content;
while (true) {
if (!(ch = cookedNextChar.Parse(state))) {
@ -331,8 +325,7 @@ struct BOZLiteral {
}
if (content.empty()) {
state->messages()->Add(
Message{at, "no digit in BOZ literal", state->context()});
state->PutMessage(at, "no digit in BOZ literal");
return {};
}
@ -340,15 +333,13 @@ struct BOZLiteral {
for (auto digit : content) {
digit = HexadecimalDigitValue(digit);
if ((digit >> *shift) > 0) {
state->messages()->Add(
Message{at, "bad digit in BOZ literal", state->context()});
state->PutMessage(at, "bad digit in BOZ literal");
return {};
}
std::uint64_t was{value};
value <<= *shift;
if ((value >> *shift) != was) {
state->messages()->Add(
Message{at, "excessive digits in BOZ literal", state->context()});
state->PutMessage(at, "excessive digits in BOZ literal");
return {};
}
value |= digit;
@ -362,7 +353,7 @@ struct DigitString {
using resultType = std::uint64_t;
static std::optional<std::uint64_t> Parse(ParseState *state) {
static constexpr auto getDigit = attempt(digit);
Position at{state->position()};
auto at = state->GetLocation();
std::optional<char> firstDigit{getDigit.Parse(state)};
if (!firstDigit) {
return {};
@ -381,8 +372,7 @@ struct DigitString {
value += digitValue;
}
if (overflow) {
state->messages()->Add(
Message{at, "overflow in decimal literal", state->context()});
state->PutMessage(at, "overflow in decimal literal");
}
return {value};
}
@ -395,7 +385,7 @@ struct HollerithLiteral {
if (!spaces.Parse(state)) {
return {};
}
Position at{state->position()};
auto at = state->GetLocation();
std::optional<std::uint64_t> charCount{DigitString{}.Parse(state)};
if (!charCount || *charCount < 1) {
return {};
@ -409,8 +399,7 @@ struct HollerithLiteral {
for (auto j = *charCount; j-- > 0;) {
std::optional<char> ch{cookedNextChar.Parse(state)};
if (!ch || !isprint(*ch)) {
state->messages()->Add(Message{at,
"insufficient or bad characters in Hollerith", state->context()});
state->PutMessage(at, "insufficient or bad characters in Hollerith");
state->set_inCharLiteral(false);
return {};
}

3
flang/lib/parser/debug-parser.h
View File

@ -23,7 +23,8 @@ public:
if (auto context = state->context()) {
std::cout << *context;
}
std::cout << state->position() << ' ' << std::string{str_, length_} << '\n';
state->GetAllSources().Identify(std::cout, state->GetProvenance(), "");
std::cout << " parser debug: " << std::string{str_, length_} << '\n';
return {Success{}};
}

13
flang/lib/parser/grammar.h
View File

@ -180,7 +180,7 @@ template<typename PA>
using statementConstructor = construct<Statement<typename PA::resultType>>;
template<typename PA> inline constexpr auto unterminatedStatement(const PA &p) {
return skipMany("\n"_tok) >> statementConstructor<PA>{}(getPosition,
return skipMany("\n"_tok) >> statementConstructor<PA>{}(getProvenance,
maybe(label), isLabelOk, spaces >> p);
}
@ -553,7 +553,7 @@ TYPE_CONTEXT_PARSER("execution part construct",
statement(indirect(dataStmt))) ||
extension(construct<ExecutionPartConstruct>{}(
statement(indirect(Parser<NamelistStmt>{})))),
construct<ExecutionPartConstruct>{}(executionPartErrorRecovery)));
construct<ExecutionPartConstruct>{}(executionPartErrorRecovery)))
// R509 execution-part -> executable-construct [execution-part-construct]...
constexpr auto executionPart =
@ -1674,7 +1674,7 @@ constexpr auto vectorSubscript = intExpr;
// "vector-subscript" is deferred to semantic analysis.
TYPE_PARSER(construct<SectionSubscript>{}(Parser<SubscriptTriplet>{}) ||
construct<SectionSubscript>{}(vectorSubscript) ||
construct<SectionSubscript>{}(subscript));
construct<SectionSubscript>{}(subscript))
// R921 subscript-triplet -> [subscript] : [subscript] [: stride]
TYPE_PARSER(construct<SubscriptTriplet>{}(
@ -3412,12 +3412,11 @@ std::optional<FunctionReference> Parser<FunctionReference>::Parse(
return {FunctionReference{std::move(call.value())}};
}
}
state->messages()->Add(
Message{state->position(), "expected (arguments)", state->context()});
state->PutMessage("expected (arguments)");
}
state->PopContext();
return {};
};
}
// R1521 call-stmt -> CALL procedure-designator [( [actual-arg-spec-list] )]
template<> std::optional<CallStmt> Parser<CallStmt>::Parse(ParseState *state) {
@ -3435,7 +3434,7 @@ template<> std::optional<CallStmt> Parser<CallStmt>::Parse(ParseState *state) {
}
}
return {};
};
}
// R1522 procedure-designator ->
// procedure-name | proc-component-ref | data-ref % binding-name

22
flang/lib/parser/message.cc
View File

@ -3,24 +3,18 @@
namespace Fortran {
namespace parser {
std::ostream &operator<<(std::ostream &o, const Message &msg) {
if (msg.context()) {
o << *msg.context();
void Message::Emit(std::ostream &o, const AllSources &sources) const {
if (context_) {
context_->Emit(o, sources);
}
o << "at line " << msg.position().lineNumber();
int column = msg.position().column();
if (column > 0) {
o << "(column " << column << ")";
}
o << ": " << msg.message() << '\n';
return o;
sources.Identify(o, provenance_, "");
o << ": " << message_ << '\n';
}
std::ostream &operator<<(std::ostream &o, const Messages &ms) {
for (const auto &msg : ms) {
o << msg;
void Messages::Emit(std::ostream &o, const AllSources &sources) const {
for (const auto &msg : messages_) {
msg.Emit(o, sources);
}
return o;
}
} // namespace parser
} // namespace Fortran

35
flang/lib/parser/message.h
View File

@ -5,7 +5,7 @@
// Supports nested contextualization.
#include "idioms.h"
#include "position.h"
#include "provenance.h"
#include <forward_list>
#include <memory>
#include <optional>
@ -24,28 +24,28 @@ public:
Message(const Message &) = default;
Message(Message &&) = default;
Message(Position pos, const std::string &msg, MessageContext ctx = nullptr)
: position_{pos}, message_{msg}, context_{ctx} {}
Message(Position pos, std::string &&msg, MessageContext ctx = nullptr)
: position_{pos}, message_{std::move(msg)}, context_{ctx} {}
Message(Position pos, const char *msg, MessageContext ctx = nullptr)
: position_{pos}, message_{msg}, context_{ctx} {}
Message(Position pos, char ch, MessageContext ctx = nullptr)
: position_{pos}, message_{"expected '"s + ch + '\''}, context_{ctx} {}
Message(Provenance at, const std::string &msg, MessageContext ctx = nullptr)
: provenance_{at}, message_{msg}, context_{ctx} {}
Message(Provenance at, std::string &&msg, MessageContext ctx = nullptr)
: provenance_{at}, message_{std::move(msg)}, context_{ctx} {}
Message(Provenance at, const char *msg, MessageContext ctx = nullptr)
: provenance_{at}, message_{msg}, context_{ctx} {}
Message &operator=(const Message &that) = default;
Message &operator=(Message &&that) = default;
Position position() const { return position_; }
bool operator<(const Message &that) const {
return provenance_ < that.provenance_;
}
Provenance provenance() const { return provenance_; }
std::string message() const { return message_; }
MessageContext context() const { return context_; }
bool operator<(const Message &that) const {
return position_ < that.position_;
}
void Emit(std::ostream &, const AllSources &) const;
private:
Position position_;
Provenance provenance_;
std::string message_;
MessageContext context_;
};
@ -77,7 +77,7 @@ public:
const_iterator cbegin() const { return messages_.cbegin(); }
const_iterator cend() const { return messages_.cend(); }
void Add(Message &&m) {
void Put(Message &&m) {
if (messages_.empty()) {
messages_.emplace_front(std::move(m));
last_ = messages_.begin();
@ -97,13 +97,12 @@ public:
}
}
void Emit(std::ostream &, const AllSources &) const;
private:
list_type messages_;
iterator last_; // valid iff messages_ nonempty
};
std::ostream &operator<<(std::ostream &, const Message &);
std::ostream &operator<<(std::ostream &, const Messages &);
} // namespace parser
} // namespace Fortran
#endif // FORTRAN_MESSAGE_H_

102
flang/lib/parser/parse-state.h
View File

@ -2,14 +2,14 @@
#define FORTRAN_PARSE_STATE_H_
// Defines the ParseState type used as the argument for every parser's
// Parse member or static function. Tracks position, context, accumulated
// messages, and an arbitrary UserState instance for parsing attempts.
// Must be efficient to duplicate and assign for backtracking and recovery
// during parsing!
// Parse member or static function. Tracks source provenance, context,
// accumulated messages, and an arbitrary UserState instance for parsing
// attempts. Must be efficient to duplicate and assign for backtracking
// and recovery during parsing!
#include "idioms.h"
#include "message.h"
#include "position.h"
#include "provenance.h"
#include <cstring>
#include <list>
#include <memory>
@ -23,11 +23,11 @@ class UserState;
class ParseState {
public:
ParseState() {}
ParseState(const char *str) : p_{str}, remaining_{std::strlen(str)} {}
ParseState(const char *str, size_t bytes) : p_{str}, remaining_{bytes} {}
ParseState(const CookedSource &cooked)
: cooked_{cooked}, p_{&cooked[0]}, remaining_{cooked.size()} {}
ParseState(const ParseState &that)
: p_{that.p_}, remaining_{that.remaining_}, position_{that.position_},
: cooked_{that.cooked_}, p_{that.p_},
remaining_{that.remaining_}, column_{that.column_},
userState_{that.userState_}, inCharLiteral_{that.inCharLiteral_},
inFortran_{that.inFortran_}, inFixedForm_{that.inFixedForm_},
enableOldDebugLines_{that.enableOldDebugLines_}, columns_{that.columns_},
@ -41,7 +41,8 @@ public:
anyErrorRecovery_{that.anyErrorRecovery_}, prescanned_{that.prescanned_} {
}
ParseState(ParseState &&that)
: p_{that.p_}, remaining_{that.remaining_}, position_{that.position_},
: cooked_{that.cooked_}, p_{that.p_},
remaining_{that.remaining_}, column_{that.column_},
messages_{std::move(that.messages_)}, context_{std::move(that.context_)},
userState_{that.userState_}, inCharLiteral_{that.inCharLiteral_},
inFortran_{that.inFortran_}, inFixedForm_{that.inFixedForm_},
@ -68,14 +69,13 @@ public:
std::memcpy(&that, buffer, bytes);
}
Position position() const { return position_; }
bool anyErrorRecovery() const { return anyErrorRecovery_; }
void set_anyErrorRecovery() { anyErrorRecovery_ = true; }
UserState *userState() const { return userState_; }
void set_userState(UserState *u) { userState_ = u; }
int column() const { return column_; }
Messages *messages() { return &messages_; }
MessageContext context() const { return context_; }
@ -85,22 +85,6 @@ public:
return was;
}
void PushContext(const std::string &str) {
context_ = std::make_shared<Message>(position_, str, context_);
}
void PushContext(std::string &&str) {
context_ = std::make_shared<Message>(position_, std::move(str), context_);
}
void PushContext(const char *str) {
context_ = std::make_shared<Message>(position_, str, context_);
}
void PopContext() {
if (context_) {
context_ = context_->context();
}
}
bool inCharLiteral() const { return inCharLiteral_; }
bool set_inCharLiteral(bool yes) {
bool was{inCharLiteral_};
@ -169,11 +153,56 @@ public:
int skippedNewLines() const { return skippedNewLines_; }
void set_skippedNewLines(int n) { skippedNewLines_ = n; }
bool prescanned() const { return prescanned_; }
void set_prescanned(bool yes) { prescanned_ = yes; }
bool prescanned() const { return prescanned_; } // TODO: always true, remove
bool tabInCurrentLine() const { return tabInCurrentLine_; }
const AllSources &GetAllSources() const { return cooked_.sources(); }
const char *GetLocation() const { return p_; }
Provenance GetProvenance(const char *at) const {
return cooked_.GetProvenance(at).start;
}
Provenance GetProvenance() const { return GetProvenance(p_); }
void PushContext(const std::string &str) {
context_ = std::make_shared<Message>(GetProvenance(), str, context_);
}
void PushContext(std::string &&str) {
context_ =
std::make_shared<Message>(GetProvenance(), std::move(str), context_);
}
void PushContext(const char *str) {
context_ = std::make_shared<Message>(GetProvenance(), str, context_);
}
void PopContext() {
if (context_) {
context_ = context_->context();
}
}
void PutMessage(Provenance at, const std::string &msg) {
messages_.Put(Message{at, msg, context_});
}
void PutMessage(const char *at, const std::string &msg) {
PutMessage(GetProvenance(at), msg);
}
void PutMessage(const std::string &msg) { PutMessage(p_, msg); }
void PutMessage(Provenance at, std::string &&msg) {
messages_.Put(Message{at, std::move(msg), context_});
}
void PutMessage(const char *at, std::string &&msg) {
PutMessage(GetProvenance(at), std::move(msg));
}
void PutMessage(std::string &&msg) { PutMessage(p_, std::move(msg)); }
void PutMessage(Provenance at, const char *msg) {
PutMessage(at, std::string{msg});
}
void PutMessage(const char *at, const char *msg) {
PutMessage(GetProvenance(at), msg);
}
void PutMessage(const char *msg) { PutMessage(p_, msg); }
bool IsAtEnd() const { return remaining_ == 0; }
std::optional<char> GetNextRawChar() const {
@ -187,24 +216,25 @@ public:
CHECK(remaining_ > 0);
--remaining_;
if (*p_ == '\n') {
position_.AdvanceLine();
column_ = 1;
tabInCurrentLine_ = false;
} else if (*p_ == '\t') {
position_.TabAdvanceColumn();
column_ = ((column_ + 7) & -8) + 1;
tabInCurrentLine_ = true;
} else {
position_.AdvanceColumn();
++column_;
}
++p_;
}
void AdvancePositionForPadding() { position_.AdvanceColumn(); }
void AdvanceColumnForPadding() { ++column_; }
private:
// Text remaining to be parsed
const CookedSource &cooked_;
const char *p_{nullptr};
size_t remaining_{0};
Position position_;
int column_{1};
// Accumulated messages and current nested context.
Messages messages_;
@ -224,7 +254,7 @@ private:
int skippedNewLines_{0};
bool tabInCurrentLine_{false};
bool anyErrorRecovery_{false};
bool prescanned_{false};
bool prescanned_{true};
// NOTE: Any additions or modifications to these data members must also be
// reflected in the copy and move constructors defined at the top of this
// class definition!

8
flang/lib/parser/parse-tree.cc
View File

@ -77,8 +77,8 @@ UNION_FORMATTER(SyncImagesStmt::ImageSet) // R1167
UNION_FORMATTER(EventWaitStmt::EventWaitSpec) // R1173
UNION_FORMATTER(FormTeamStmt::FormTeamSpec) // R1177
UNION_FORMATTER(LockStmt::LockStat) // R1179
UNION_FORMATTER(IoUnit); // R1201, R1203
UNION_FORMATTER(ConnectSpec); // R1205
UNION_FORMATTER(IoUnit) // R1201, R1203
UNION_FORMATTER(ConnectSpec) // R1205
UNION_FORMATTER(CloseStmt::CloseSpec) // R1209
UNION_FORMATTER(IoControlSpec) // R1213
UNION_FORMATTER(Format) // R1215
@ -90,7 +90,7 @@ UNION_FORMATTER(EndfileStmt) // R1225
UNION_FORMATTER(RewindStmt) // R1226
UNION_FORMATTER(PositionOrFlushSpec) // R1227 & R1229
UNION_FORMATTER(FlushStmt) // R1228
UNION_FORMATTER(InquireStmt); // R1230
UNION_FORMATTER(InquireStmt) // R1230
UNION_FORMATTER(InquireSpec) // R1231
UNION_FORMATTER(ModuleSubprogram) // R1408
UNION_FORMATTER(Rename) // R1411
@ -733,7 +733,7 @@ ProcedureDesignator Designator::ConvertToProcedureDesignator() {
std::optional<Call> Designator::ConvertToCall() {
return std::visit(
visitors{[](ObjectName &n) -> std::optional<Call> {
return {Call{ProcedureDesignator{std : move(n)},
return {Call{ProcedureDesignator{std::move(n)},
std::list<ActualArgSpec>{}}};
},
[this](DataReference &dr) -> std::optional<Call> {

23
flang/lib/parser/parse-tree.h
View File

@ -13,7 +13,7 @@
#include "idioms.h"
#include "indirection.h"
#include "message.h"
#include "position.h"
#include "provenance.h"
#include <cinttypes>
#include <list>
#include <optional>
@ -60,22 +60,22 @@
// Many classes below simply wrap a std::variant<> discriminated union,
// which is conventionally named "u".
#define UNION_CLASS_BOILERPLATE(classname) \
BOILERPLATE(classname); \
template<typename A> classname(A &&x) : u(std::move(x)) {}
template<typename A> classname(A &&x) : u(std::move(x)) {} \
BOILERPLATE(classname)
// Many other classes below simply wrap a std::tuple<> structure, which
// is conventionally named "t".
#define TUPLE_CLASS_BOILERPLATE(classname) \
BOILERPLATE(classname); \
template<typename... Ts> \
classname(Ts &&... args) : t(std::forward<Ts>(args)...) {}
classname(Ts &&... args) : t(std::forward<Ts>(args)...) {} \
BOILERPLATE(classname)
// Many other classes below simply wrap a single data member, which is
// conventionally named "v".
#define WRAPPER_CLASS_BOILERPLATE(classname, type) \
BOILERPLATE(classname); \
classname(type &&x) : v(std::move(x)) {} \
type v;
type v
#define WRAPPER_CLASS(classname, type) \
struct classname { \
@ -275,13 +275,12 @@ using ScalarDefaultCharConstantExpr = Scalar<DefaultChar<ConstantExpr>>;
using Label = std::uint64_t; // validated later, must be in [1..99999]
// A wrapper for xzy-stmt productions that are statements, so that
// source positions and labels have a uniform representation.
// source provenances and labels have a uniform representation.
template<typename A> struct Statement {
Statement(Position &&pos, std::optional<long> &&lab, bool &&accept, A &&s)
: position(std::move(pos)),
label(std::move(lab)), isLabelInAcceptableField{accept},
Statement(Provenance &&at, std::optional<long> &&lab, bool &&accept, A &&s)
: provenance(at), label(std::move(lab)), isLabelInAcceptableField{accept},
statement(std::move(s)) {}
Position position;
Provenance provenance;
std::optional<Label> label;
bool isLabelInAcceptableField{true};
A statement;
@ -3190,7 +3189,7 @@ template<typename A>
std::ostream &operator<<(std::ostream &o, const Statement<A> &x) {
return o << "(Statement " << x.label << ' '
<< (x.isLabelInAcceptableField ? ""s : "!isLabelInAcceptableField "s)
<< x.position << ' ' << x.statement << ')';
<< ' ' << x.statement << ')';
}
template<typename A>

9
flang/lib/parser/position.cc
View File

@ -1,9 +0,0 @@
#include "position.h"
namespace Fortran {
namespace parser {
std::ostream &operator<<(std::ostream &o, const Position &x) {
return o << "(at line " << x.lineNumber() << ", column " << x.column() << ')';
}
} // namespace parser
} // namespace Fortran

76
flang/lib/parser/position.h
View File

@ -1,76 +0,0 @@
#ifndef FORTRAN_POSITION_H_
#define FORTRAN_POSITION_H_
// Represents a position in a source file.
// TODO: Support multiple source files for inclusion and support contextual
// positions for macro expansions.
#include <ostream>
namespace Fortran {
namespace parser {
class Position {
public:
constexpr Position() {}
constexpr Position(const Position &) = default;
constexpr Position(Position &&) = default;
constexpr Position(int ln, int col) : lineNumber_{ln}, column_{col} {}
constexpr Position &operator=(const Position &) = default;
constexpr Position &operator=(Position &&) = default;
constexpr int lineNumber() const { return lineNumber_; }
constexpr int column() const { return column_; }
Position &set_lineNumber(int line) {
lineNumber_ = line;
return *this;
}
Position &set_column(int column) {
column_ = column;
return *this;
}
constexpr bool operator<(const Position &that) const {
return lineNumber_ < that.lineNumber_ ||
(lineNumber_ == that.lineNumber_ && column_ < that.column_);
}
constexpr bool operator<=(const Position &that) const {
return lineNumber_ < that.lineNumber_ ||
(lineNumber_ == that.lineNumber_ && column_ <= that.column_);
}
constexpr bool operator==(const Position &that) const {
return lineNumber_ == that.lineNumber_ && column_ == that.column_;
}
constexpr bool operator!=(const Position &that) const {
return !operator==(that);
}
constexpr bool operator>(const Position &that) const {
return !operator<=(that);
}
constexpr bool operator>=(const Position &that) const {
return !operator<(that);
}
void AdvanceColumn() { ++column_; }
void TabAdvanceColumn() { column_ = ((column_ + 7) & -8) + 1; }
void AdvanceLine() {
++lineNumber_;
column_ = 1;
}
private:
int lineNumber_{1};
int column_{1};
};
std::ostream &operator<<(std::ostream &, const Position &);
} // namespace parser
} // namespace Fortran
#endif // FORTRAN_POSITION_H_

181
flang/lib/parser/preprocessor.cc
View File

@ -1,5 +1,4 @@
#include "preprocessor.h"
#include "char-buffer.h"
#include "idioms.h"
#include "prescan.h"
#include <algorithm>
@ -25,7 +24,27 @@ bool CharPointerWithLength::IsBlank() const {
return true;
}
void TokenSequence::Append(const TokenSequence &that) {
void TokenSequence::clear() {
start_.clear();
nextStart_ = 0;
char_.clear();
}
void TokenSequence::pop_back() {
size_t bytes{nextStart_ - start_.back()};
nextStart_ = start_.back();
start_.pop_back();
char_.resize(nextStart_);
provenances_.RemoveLastBytes(bytes);
}
void TokenSequence::shrink_to_fit() {
start_.shrink_to_fit();
char_.shrink_to_fit();
provenances_.shrink_to_fit();
}
void TokenSequence::Put(const TokenSequence &that) {
if (nextStart_ < char_.size()) {
start_.push_back(nextStart_);
}
@ -35,9 +54,45 @@ void TokenSequence::Append(const TokenSequence &that) {
}
char_.insert(char_.end(), that.char_.begin(), that.char_.end());
nextStart_ = char_.size();
provenances_.Put(that.provenances_);
}
void TokenSequence::EmitWithCaseConversion(CharBuffer *out) const {
void TokenSequence::Put(const TokenSequence &that, size_t at, size_t tokens) {
ProvenanceRange provenance;
size_t offset{0};
for (; tokens-- > 0; ++at) {
CharPointerWithLength tok{that[at]};
size_t tokBytes{tok.size()};
for (size_t j{0}; j < tokBytes; ++j) {
if (offset == provenance.bytes) {
offset = 0;
provenance = that.provenances_.Map(that.start_[at] + j);
}
PutNextTokenChar(tok[j], provenance.start + offset++);
}
CloseToken();
}
}
void TokenSequence::Put(const char *s, size_t bytes, Provenance provenance) {
for (size_t j{0}; j < bytes; ++j) {
PutNextTokenChar(s[j], provenance++);
}
CloseToken();
}
void TokenSequence::Put(const CharPointerWithLength &t, Provenance provenance) {
Put(&t[0], t.size(), provenance);
}
void TokenSequence::Put(const std::string &s, Provenance provenance) {
Put(s.data(), s.size(), provenance);
}
void TokenSequence::Put(const std::stringstream &ss, Provenance provenance) {
Put(ss.str(), provenance);
}
void TokenSequence::EmitWithCaseConversion(CookedSource *cooked) const {
size_t tokens{start_.size()};
size_t chars{char_.size()};
size_t atToken{0};
@ -45,52 +100,24 @@ void TokenSequence::EmitWithCaseConversion(CharBuffer *out) const {
size_t nextStart{atToken + 1 < tokens ? start_[++atToken] : chars};
if (isalpha(char_[j])) {
for (; j < nextStart; ++j) {
out->Put(tolower(char_[j]));
cooked->Put(tolower(char_[j]));
}
} else {
out->Put(&char_[j], nextStart - j);
cooked->Put(&char_[j], nextStart - j);
j = nextStart;
}
}
cooked->Put(provenances_);
}
std::string TokenSequence::ToString() const {
return {&char_[0], char_.size()};
}
void TokenSequence::clear() {
start_.clear();
nextStart_ = 0;
char_.clear();
}
void TokenSequence::push_back(const char *s, size_t bytes) {
for (size_t j{0}; j < bytes; ++j) {
AddChar(s[j]);
}
EndToken();
}
void TokenSequence::push_back(const CharPointerWithLength &t) {
push_back(&t[0], t.size());
}
void TokenSequence::push_back(const std::string &s) {
push_back(s.data(), s.size());
}
void TokenSequence::push_back(const std::stringstream &ss) {
push_back(ss.str());
}
void TokenSequence::pop_back() {
nextStart_ = start_.back();
start_.pop_back();
char_.resize(nextStart_);
}
void TokenSequence::shrink_to_fit() {
start_.shrink_to_fit();
char_.shrink_to_fit();
ProvenanceRange TokenSequence::GetProvenance(
size_t token, size_t offset) const {
ProvenanceRange range{provenances_.Map(start_[token] + offset)};
return {range.start, std::min(range.bytes, TokenBytes(token) - offset)};
}
Definition::Definition(
@ -135,11 +162,11 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
if (IsIdentifierFirstCharacter(tok)) {
auto it = args.find(tok.ToString());
if (it != args.end()) {
result.push_back(it->second);
result.Put(it->second, 0);
continue;
}
}
result.push_back(tok);
result.Put(token, j, 1);
}
return result;
}
@ -168,36 +195,37 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
if (index >= args.size()) {
continue;
}
int lastNonBlank{static_cast<int>(result.size()) - 1};
for (; lastNonBlank >= 0; --lastNonBlank) {
if (!result[lastNonBlank].IsBlank()) {
size_t afterLastNonBlank{result.size()};
for (; afterLastNonBlank > 0; --afterLastNonBlank) {
if (!result[afterLastNonBlank - 1].IsBlank()) {
break;
}
}
size_t argTokens{args[index].size()};
if (lastNonBlank >= 0 && result[lastNonBlank].ToString() == "#") {
while (result.size() > static_cast<size_t>(lastNonBlank)) {
if (afterLastNonBlank > 0 &&
result[afterLastNonBlank - 1].ToString() == "#") {
while (result.size() >= afterLastNonBlank) {
result.pop_back();
}
std::string strung{'"'};
result.PutNextTokenChar('"', 0); // TODO provenance
for (size_t k{0}; k < argTokens; ++k) {
const CharPointerWithLength &arg{args[index][k]};
size_t argBytes{args[index][k].size()};
for (size_t n{0}; n < argBytes; ++n) {
char ch{arg[n]};
Provenance from{args[index].GetProvenance(k, n).start};
if (ch == '"' || ch == '\\') {
strung += ch;
result.PutNextTokenChar(ch, from);
}
strung += ch;
result.PutNextTokenChar(ch, from);
}
}
result.push_back(strung + '"');
result.PutNextTokenChar('"', 0); // TODO provenance
result.CloseToken();
} else {
for (size_t k{0}; k < argTokens; ++k) {
const CharPointerWithLength &argToken{args[index][k]};
if (pasting && argToken.IsBlank()) {
} else {
result.push_back(argToken);
if (!pasting || !args[index][k].IsBlank()) {
result.Put(args[index], k);
pasting = false;
}
}
@ -218,11 +246,9 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
token.ToString() == "__VA_ARGS__") {
for (size_t k{argumentCount_}; k < args.size(); ++k) {
if (k > argumentCount_) {
result.push_back(","s);
}
for (size_t n{0}; n < args[k].size(); ++n) {
result.push_back(args[k][n]);
result.Put(","s, 0); // TODO provenance
}
result.Put(args[k]);
}
} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
j + 2 < tokens && replacement_[j + 1].ToString() == "(" &&
@ -239,7 +265,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args) {
continue;
}
}
result.push_back(token);
result.Put(replacement_, j);
}
}
return result;
@ -280,42 +306,39 @@ bool Preprocessor::MacroReplacement(
if (j == tokens) {
return false; // nothing appeared that could be replaced
}
for (size_t k{0}; k < j; ++k) {
result->push_back(input[k]);
}
result->Put(input, 0, j);
for (; j < tokens; ++j) {
const CharPointerWithLength &token{input[j]};
if (token.IsBlank() || !IsIdentifierFirstCharacter(token[0])) {
result->push_back(token);
result->Put(input, j);
continue;
}
auto it = definitions_.find(token);
if (it == definitions_.end()) {
result->push_back(token);
result->Put(input, j);
continue;
}
Definition &def{it->second};
if (def.isDisabled()) {
result->push_back(token);
result->Put(input, j);
continue;
}
if (!def.isFunctionLike()) {
if (def.isPredefined()) {
std::string name{def.replacement()[0].ToString()};
if (name == "__FILE__") {
result->Append("\""s + prescanner_.sourceFile().path() + '"');
result->Put("\""s + prescanner_.GetCurrentPath() + '"');
continue;
}
if (name == "__LINE__") {
std::stringstream ss;
ss << prescanner_.position().lineNumber();
result->Append(ss.str());
ss << prescanner_.GetCurrentLineNumber();
result->Put(ss.str());
continue;
}
}
def.set_isDisabled(true);
result->Append(ReplaceMacros(def.replacement()));
result->Put(ReplaceMacros(def.replacement()));
def.set_isDisabled(false);
continue;
}
@ -331,7 +354,7 @@ bool Preprocessor::MacroReplacement(
}
}
if (!leftParen) {
result->push_back(token);
result->Put(input, j);
continue;
}
std::vector<size_t> argStart{++k};
@ -352,7 +375,7 @@ bool Preprocessor::MacroReplacement(
}
if (k >= tokens || argStart.size() < def.argumentCount() ||
(argStart.size() > def.argumentCount() && !def.isVariadic())) {
result->push_back(token);
result->Put(input, j);
continue;
}
j = k; // advance to the terminal ')'
@ -360,14 +383,10 @@ bool Preprocessor::MacroReplacement(
for (k = 0; k < argStart.size(); ++k) {
size_t at{argStart[k]};
size_t count{(k + 1 == argStart.size() ? j : argStart[k + 1] - 1) - at};
TokenSequence actual;
for (; count-- > 0; ++at) {
actual.push_back(input[at]);
}
args.emplace_back(std::move(actual));
args.emplace_back(TokenSequence(input, at, count));
}
def.set_isDisabled(true);
result->Append(ReplaceMacros(def.Apply(args)));
result->Put(ReplaceMacros(def.Apply(args)));
def.set_isDisabled(false);
}
return true;
@ -393,7 +412,7 @@ static TokenSequence StripBlanks(
TokenSequence noBlanks;
for (size_t j{SkipBlanks(token, first, tokens)}; j < tokens;
j = SkipBlanks(token, j + 1, tokens)) {
noBlanks.push_back(token[j]);
noBlanks.Put(token, j);
}
return noBlanks;
}
@ -628,7 +647,7 @@ bool Preprocessor::SkipDisabledConditionalCode(
}
void Preprocessor::Complain(const std::string &message) {
prescanner_.messages().Add({prescanner_.position(), message});
prescanner_.messages().Put({prescanner_.GetCurrentProvenance(), message});
}
// Precedence level codes used here to accommodate mixed Fortran and C:
@ -918,11 +937,11 @@ bool Preprocessor::IsIfPredicateTrue(
name = expr1[j++];
}
if (!name.empty()) {
expr2.push_back(IsNameDefined(name) ? "1" : "0", 1);
expr2.Put(IsNameDefined(name) ? "1" : "0", 1, 0); // TODO provenance
continue;
}
}
expr2.push_back(expr1[j]);
expr2.Put(expr1, j);
}
TokenSequence expr3{ReplaceMacros(expr2)};
TokenSequence expr4{StripBlanks(expr3, 0, expr3.size())};

61
flang/lib/parser/preprocessor.h
View File

@ -8,6 +8,7 @@
// extensions for preprocessing will not be necessary.
#include "idioms.h"
#include "provenance.h"
#include <cctype>
#include <cstring>
#include <functional>
@ -21,7 +22,7 @@
namespace Fortran {
namespace parser {
class CharBuffer;
class CookedSource;
class Prescanner;
// Just a const char pointer with an associated length; does not own the
@ -78,19 +79,22 @@ namespace Fortran {
namespace parser {
// Buffers a contiguous sequence of characters that has been partitioned into
// a sequence of preprocessing tokens.
// a sequence of preprocessing tokens with provenances.
class TokenSequence {
public:
TokenSequence() {}
TokenSequence(const TokenSequence &that) { Append(that); }
TokenSequence(const TokenSequence &that) { Put(that); }
TokenSequence(const TokenSequence &that, size_t at, size_t count = 1) {
Put(that, at, count);
}
TokenSequence(TokenSequence &&that)
: start_{std::move(that.start_)},
nextStart_{that.nextStart_}, char_{std::move(that.char_)} {}
TokenSequence(const std::string &s) { push_back(s); }
TokenSequence(const std::string &s) { Put(s, 0); } // TODO predefined prov.
TokenSequence &operator=(const TokenSequence &that) {
clear();
Append(that);
Put(that);
return *this;
}
TokenSequence &operator=(TokenSequence &&that) {
@ -101,14 +105,22 @@ public:
}
CharPointerWithLength operator[](size_t token) const {
return {&char_[start_[token]],
(token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
start_[token]};
return {&char_[start_[token]], TokenBytes(token)};
}
void AddChar(char ch) { char_.emplace_back(ch); }
bool empty() const { return start_.empty(); }
size_t size() const { return start_.size(); }
const char *data() const { return &char_[0]; }
void clear();
void pop_back();
void shrink_to_fit();
void EndToken() {
void PutNextTokenChar(char ch, Provenance provenance) {
char_.emplace_back(ch);
provenances_.Put({provenance, 1});
}
void CloseToken() {
// CHECK(char_.size() > nextStart_);
start_.emplace_back(nextStart_);
nextStart_ = char_.size();
@ -119,25 +131,26 @@ public:
start_.pop_back();
}
void Append(const TokenSequence &);
void EmitWithCaseConversion(CharBuffer *) const;
void Put(const TokenSequence &);
void Put(const TokenSequence &, size_t at, size_t tokens = 1);
void Put(const char *, size_t, Provenance);
void Put(const CharPointerWithLength &, Provenance);
void Put(const std::string &, Provenance);
void Put(const std::stringstream &, Provenance);
void EmitWithCaseConversion(CookedSource *) const;
std::string ToString() const;
bool empty() const { return start_.empty(); }
size_t size() const { return start_.size(); }
const char *data() const { return &char_[0]; }
void clear();
void push_back(const char *, size_t);
void push_back(const CharPointerWithLength &);
void push_back(const std::string &);
void push_back(const std::stringstream &);
void pop_back();
void shrink_to_fit();
ProvenanceRange GetProvenance(size_t token, size_t offset = 0) const;
private:
std::vector<int> start_;
size_t TokenBytes(size_t token) const {
return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
start_[token];
}
std::vector<size_t> start_;
size_t nextStart_{0};
std::vector<char> char_;
OffsetToProvenanceMappings provenances_;
};
// Defines a macro

98
flang/lib/parser/prescan.cc
View File

@ -1,5 +1,4 @@
#include "prescan.h"
#include "char-buffer.h"
#include "idioms.h"
#include "source.h"
#include <cctype>
@ -10,12 +9,13 @@
namespace Fortran {
namespace parser {
CharBuffer Prescanner::Prescan(const SourceFile &source) {
sourceFile_ = &source;
lineStart_ = source.content();
limit_ = lineStart_ + source.bytes();
CharBuffer out;
CookedSource Prescanner::Prescan() {
startProvenance_ = 0;
start_ = &allSources_[0];
limit_ = start_ + allSources_.size();
lineStart_ = start_;
TokenSequence tokens, preprocessed;
CookedSource cooked{allSources_};
while (lineStart_ < limit_) {
if (CommentLinesAndPreprocessorDirectives() && lineStart_ >= limit_) {
break;
@ -29,26 +29,26 @@ CharBuffer Prescanner::Prescan(const SourceFile &source) {
while (NextToken(&tokens)) {
}
if (preprocessor_.MacroReplacement(tokens, &preprocessed)) {
preprocessed.AddChar('\n');
preprocessed.EndToken();
EmitChar(&preprocessed, '\n');
preprocessed.CloseToken();
if (IsFixedFormCommentLine(preprocessed.data()) ||
IsFreeFormComment(preprocessed.data())) {
++newlineDebt_;
} else {
preprocessed.pop_back(); // clip the newline added above
preprocessed.EmitWithCaseConversion(&out);
preprocessed.EmitWithCaseConversion(&cooked);
}
preprocessed.clear();
} else {
tokens.EmitWithCaseConversion(&out);
tokens.EmitWithCaseConversion(&cooked);
}
tokens.clear();
out.Put('\n');
PayNewlineDebt(&out);
cooked.Put('\n', 0);
PayNewlineDebt(&cooked);
}
PayNewlineDebt(&out);
sourceFile_ = nullptr;
return std::move(out);
PayNewlineDebt(&cooked);
cooked.Marshal();
return cooked;
}
std::optional<TokenSequence> Prescanner::NextTokenizedLine() {
@ -57,7 +57,6 @@ std::optional<TokenSequence> Prescanner::NextTokenizedLine() {
}
bool wasInPreprocessorDirective{inPreprocessorDirective_};
auto saveAt = at_;
auto saveAtPosition = atPosition_;
inPreprocessorDirective_ = true;
BeginSourceLineAndAdvance();
TokenSequence tokens;
@ -65,10 +64,17 @@ std::optional<TokenSequence> Prescanner::NextTokenizedLine() {
}
inPreprocessorDirective_ = wasInPreprocessorDirective;
at_ = saveAt;
atPosition_ = saveAtPosition;
return {std::move(tokens)};
}
std::string Prescanner::GetCurrentPath() const {
return allSources_.GetPath(GetCurrentProvenance());
}
int Prescanner::GetCurrentLineNumber() const {
return allSources_.GetLineNumber(GetCurrentProvenance());
}
void Prescanner::NextLine() {
void *vstart{static_cast<void *>(const_cast<char *>(lineStart_))};
void *v{std::memchr(vstart, '\n', limit_ - lineStart_)};
@ -78,49 +84,47 @@ void Prescanner::NextLine() {
const char *nl{const_cast<const char *>(static_cast<char *>(v))};
lineStart_ = nl + 1;
}
lineStartPosition_.AdvanceLine();
}
void Prescanner::LabelField(TokenSequence *token) {
int outCol{1};
for (; *at_ != '\n' && atPosition_.column() <= 6; ++at_) {
for (; *at_ != '\n' && column_ <= 6; ++at_) {
if (*at_ == '\t') {
++at_;
atPosition_.set_column(7);
column_ = 7;
break;
}
if (*at_ != ' ' &&
(*at_ != '0' ||
atPosition_.column() != 6)) { // '0' in column 6 becomes space
token->AddChar(*at_);
(*at_ != '0' || column_ != 6)) { // '0' in column 6 becomes space
EmitChar(token, *at_);
++outCol;
}
atPosition_.AdvanceColumn();
++column_;
}
if (outCol > 1) {
token->EndToken();
token->CloseToken();
}
if (outCol < 7) {
for (; outCol < 7; ++outCol) {
token->AddChar(' ');
EmitChar(token, ' ');
}
token->EndToken();
token->CloseToken();
}
}
void Prescanner::NextChar() {
// CHECK(*at_ != '\n');
++at_;
atPosition_.AdvanceColumn();
++column_;
if (inPreprocessorDirective_) {
while (*at_ == '/' && at_[1] == '*') {
char star{' '}, slash{' '};
at_ += 2;
atPosition_.set_column(atPosition_.column() + 2);
column_ += 2;
while ((*at_ != '\n' || slash == '\\') && (star != '*' || slash != '/')) {
star = slash;
slash = *at_++;
atPosition_.AdvanceColumn();
++column_;
}
}
while (*at_ == '\\' && at_ + 2 < limit_ && at_[1] == '\n') {
@ -128,7 +132,7 @@ void Prescanner::NextChar() {
++newlineDebt_;
}
} else {
if ((inFixedForm_ && atPosition_.column() > fixedFormColumnLimit_ &&
if ((inFixedForm_ && column_ > fixedFormColumnLimit_ &&
!tabInCurrentLine_) ||
(*at_ == '!' && !inCharLiteral_)) {
while (*at_ != '\n') {
@ -161,11 +165,12 @@ bool Prescanner::NextToken(TokenSequence *tokens) {
if (inFixedForm_) {
SkipSpaces();
} else if (*at_ == ' ' || *at_ == '\t') {
Provenance here{GetCurrentProvenance()};
NextChar();
SkipSpaces();
if (*at_ != '\n') {
tokens->AddChar(' ');
tokens->EndToken();
tokens->PutNextTokenChar(' ', here);
tokens->CloseToken();
return true;
}
}
@ -194,7 +199,7 @@ bool Prescanner::NextToken(TokenSequence *tokens) {
inCharLiteral_ = true;
while (n-- > 0) {
if (PadOutCharacterLiteral()) {
tokens->AddChar(' ');
EmitChar(tokens, ' ');
} else {
if (*at_ == '\n') {
break; // TODO error
@ -256,7 +261,7 @@ bool Prescanner::NextToken(TokenSequence *tokens) {
EmitCharAndAdvance(tokens, nch);
}
}
tokens->EndToken();
tokens->CloseToken();
return true;
}
@ -285,12 +290,12 @@ void Prescanner::QuotedCharacterLiteral(TokenSequence *tokens) {
do {
EmitCharAndAdvance(tokens, *at_);
while (PadOutCharacterLiteral()) {
tokens->AddChar(' ');
EmitChar(tokens, ' ');
}
if (*at_ == '\\' && enableBackslashEscapesInCharLiterals_) {
EmitCharAndAdvance(tokens, '\\');
while (PadOutCharacterLiteral()) {
tokens->AddChar(' ');
EmitChar(tokens, ' ');
}
} else if (*at_ == quote) {
// A doubled quote mark becomes a single instance of the quote character
@ -309,8 +314,8 @@ void Prescanner::QuotedCharacterLiteral(TokenSequence *tokens) {
bool Prescanner::PadOutCharacterLiteral() {
if (inFixedForm_ && !tabInCurrentLine_ && *at_ == '\n' &&
atPosition_.column() < fixedFormColumnLimit_) {
atPosition_.AdvanceColumn();
column_ < fixedFormColumnLimit_) {
++column_;
return true;
}
return false;
@ -439,7 +444,7 @@ bool Prescanner::FixedFormContinuation() {
return false;
}
BeginSourceLine(cont);
atPosition_.set_column(7);
column_ = 7;
++newlineDebt_;
NextLine();
return true;
@ -463,32 +468,31 @@ bool Prescanner::FreeFormContinuation() {
if (p >= limit_) {
return false;
}
int column{1};
column_ = 1;
for (; *p == ' ' || *p == '\t'; ++p) {
++column;
++column_;
}
if (*p == '&') {
++p;
++column;
++column_;
} else if (ampersand || delimiterNesting_ > 0) {
if (p > lineStart_) {
--p;
--column;
--column_;
}
} else {
return false; // not a continuation
}
at_ = p;
(atPosition_ = lineStartPosition_).set_column(column);
tabInCurrentLine_ = false;
++newlineDebt_;
NextLine();
return true;
}
void Prescanner::PayNewlineDebt(CharBuffer *out) {
void Prescanner::PayNewlineDebt(CookedSource *cooked) {
for (; newlineDebt_ > 0; --newlineDebt_) {
out->Put('\n');
cooked->Put('\n', 0);
}
}
} // namespace parser

49
flang/lib/parser/prescan.h
View File

@ -7,29 +7,23 @@
// line continuation, comment removal, card image margins, padding out
// fixed form character literals on truncated card images, and drives the
// Fortran source preprocessor.
//
// It is possible to run the Fortran parser without running this prescan
// phase, using only the parsers defined in cooked-chars.h, so long as
// preprocessing and INCLUDE lines need not be handled.
#include "char-buffer.h"
#include "message.h"
#include "position.h"
#include "preprocessor.h"
#include "provenance.h"
#include "source.h"
#include <optional>
#include <string>
namespace Fortran {
namespace parser {
class Prescanner {
public:
explicit Prescanner(Messages &messages)
: messages_{messages}, preprocessor_{*this} {}
Prescanner(Messages &messages, AllSources &allSources)
: messages_{messages}, allSources_{allSources}, preprocessor_{*this} {}
Messages &messages() const { return messages_; }
const SourceFile &sourceFile() const { return *sourceFile_; }
Position position() const { return atPosition_; }
bool anyFatalErrors() const { return anyFatalErrors_; }
Prescanner &set_fixedForm(bool yes) {
@ -49,13 +43,15 @@ public:
return *this;
}
CharBuffer Prescan(const SourceFile &source);
CookedSource Prescan();
std::optional<TokenSequence> NextTokenizedLine();
Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
std::string GetCurrentPath() const; // __FILE__
int GetCurrentLineNumber() const; // __LINE__
private:
void BeginSourceLine(const char *at) {
at_ = at;
atPosition_ = lineStartPosition_;
tabInCurrentLine_ = false;
preventHollerith_ = false;
delimiterNesting_ = 0;
@ -66,8 +62,16 @@ private:
NextLine();
}
Provenance GetProvenance(const char *sourceChar) const {
return startProvenance_ + sourceChar - start_;
}
void EmitChar(TokenSequence *tokens, char ch) {
tokens->PutNextTokenChar(ch, GetCurrentProvenance());
}
char EmitCharAndAdvance(TokenSequence *tokens, char ch) {
tokens->AddChar(ch);
EmitChar(tokens, ch);
NextChar();
return *at_;
}
@ -88,23 +92,26 @@ private:
const char *FixedFormContinuationLine();
bool FixedFormContinuation();
bool FreeFormContinuation();
void PayNewlineDebt(CharBuffer *);
void PayNewlineDebt(CookedSource *);
Messages &messages_;
bool anyFatalErrors_{false};
const char *lineStart_{nullptr}; // next line to process; <= limit_
AllSources &allSources_;
Provenance startProvenance_;
const char *start_{nullptr}; // beginning of sourceFile_ content
const char *limit_{nullptr}; // first address after end of source
const char *at_{nullptr}; // next character to process; < lineStart_
int column_{1}; // card image column position of next character
const char *limit_{nullptr}; // first address after end of source
const char *lineStart_{nullptr}; // next line to process; <= limit_
bool tabInCurrentLine_{false};
bool preventHollerith_{false};
bool anyFatalErrors_{false};
int newlineDebt_{0}; // newline characters consumed but not yet emitted
const SourceFile *sourceFile_{nullptr};
Position atPosition_, lineStartPosition_;
bool inCharLiteral_{false};
bool inPreprocessorDirective_{false};
bool inFixedForm_{true};
int fixedFormColumnLimit_{72};
bool tabInCurrentLine_{false};
bool preventHollerith_{false};
bool enableOldDebugLines_{false};
bool enableBackslashEscapesInCharLiterals_{true};
int delimiterNesting_{0};

242
flang/lib/parser/provenance.cc
View File

@ -1,87 +1,203 @@
#include "provenance.h"
#include "idioms.h"
#include "position.h"
#include <utility>
namespace Fortran {
namespace parser {
Origin::Origin(const SourceFile &source) : u_{Inclusion{source}} {}
Origin::Origin(const SourceFile &included, ProvenanceRange from)
: u_{Inclusion{included}}, replaces_{from} {}
Origin::Origin(ProvenanceRange def, ProvenanceRange use,
const std::string &expansion)
: u_{Macro{def, expansion}}, replaces_{use} {}
size_t Origin::size() const {
return std::visit(
visitors{[](const Inclusion &inc) { return inc.source.bytes(); },
[](const Macro &mac) { return mac.expansion.size(); }},
u_);
void OffsetToProvenanceMappings::Put(ProvenanceRange range) {
if (provenanceMap_.empty()) {
provenanceMap_.push_back({bytes_, range});
} else {
ContiguousProvenanceMapping &last{provenanceMap_.back()};
if (range.start == last.range.start + last.range.bytes) {
last.range.bytes += range.bytes;
} else {
provenanceMap_.push_back({bytes_, range});
}
}
bytes_ += range.bytes;
}
const char &Origin::operator[](size_t n) const {
return std::visit(
visitors{[n](const Inclusion &inc) -> const char & {
return inc.source.content()[n]; },
[n](const Macro &mac) -> const char & { return mac.expansion[n]; }},
u_);
void OffsetToProvenanceMappings::Put(const OffsetToProvenanceMappings &that) {
for (const auto &map : that.provenanceMap_) {
Put(map.range);
}
}
void Origin::Identify(std::ostream &o, const AllOfTheSource &sources, size_t at,
const std::string &prefix) const {
static const std::string indented{prefix + " "};
std::visit(
visitors{
[&](const Inclusion &inc) {
Position pos{inc.source.FindOffsetPosition(at)};
o << prefix << "at line " << pos.lineNumber() << ", column " <<
pos.column() << "in the file " << inc.source.path() << '\n';
if (replaces_.bytes() > 0) {
o << prefix << " that was included\n";
sources.Identify(o, replaces_.start(), indented);
}
},
[&](const Macro &mac) {
o << prefix << "in the expansion of a macro that was defined\n";
sources.Identify(o, mac.definition.start(), indented);
o << prefix << "... and called\n";
sources.Identify(o, replaces_.start(), indented);
o << prefix << "... and expanded to\n" <<
indented << mac.expansion << '\n'; }},
u_);
}
AllOfTheSource &AllOfTheSource::Add(Origin &&origin) {
size_t start{bytes_};
bytes_ += origin.size();
chunk_.emplace_back(Chunk{std::move(origin), start});
return *this;
}
const char &AllOfTheSource::operator[](Provenance at) const {
const Chunk &chunk{MapToChunk(at)};
return chunk.origin[at - chunk.start];
}
const AllOfTheSource::Chunk &AllOfTheSource::MapToChunk(Provenance at) const {
ProvenanceRange OffsetToProvenanceMappings::Map(size_t at) const {
CHECK(at < bytes_);
size_t low{0}, count{chunk_.size()};
size_t low{0}, count{provenanceMap_.size()};
while (count > 1) {
size_t mid{low + (count >> 1)};
if (chunk_[mid].start > at) {
if (provenanceMap_[mid].start > at) {
count = mid - low;
} else {
count -= mid - low;
low = mid;
}
}
return chunk_[low];
size_t offset{at - provenanceMap_[low].start};
return {provenanceMap_[low].start + offset,
provenanceMap_[low].range.bytes - offset};
}
void AllOfTheSource::Identify(std::ostream &o, Provenance at,
const std::string &prefix) const {
const Chunk &chunk{MapToChunk(at)};
return chunk.origin.Identify(o, *this, at - chunk.start, prefix);
void OffsetToProvenanceMappings::RemoveLastBytes(size_t bytes) {
for (; bytes > 0; provenanceMap_.pop_back()) {
if (provenanceMap_.empty()) {
break;
}
ContiguousProvenanceMapping &last{provenanceMap_.back()};
if (bytes < last.range.bytes) {
last.range.bytes -= bytes;
break;
}
bytes -= last.range.bytes;
}
}
AllSources::AllSources(const SourceFile &initialSourceFile) {
AddIncludedFile(initialSourceFile, ProvenanceRange{});
}
const char &AllSources::operator[](Provenance at) const {
const Origin &origin{MapToOrigin(at)};
return origin[at - origin.start];
}
ProvenanceRange AllSources::AddIncludedFile(
const SourceFile &source, ProvenanceRange from) {
size_t start{bytes_}, bytes{source.bytes()};
bytes_ += bytes;
origin_.emplace_back(start, source, from);
return {start, bytes};
}
ProvenanceRange AllSources::AddMacroCall(
ProvenanceRange def, ProvenanceRange use, const std::string &expansion) {
size_t start{bytes_}, bytes{expansion.size()};
bytes_ += bytes;
origin_.emplace_back(start, def, use, expansion);
return {start, bytes};
}
ProvenanceRange AllSources::AddCompilerInsertion(const std::string &text) {
size_t start{bytes_}, bytes{text.size()};
bytes_ += bytes;
origin_.emplace_back(start, text);
return {start, bytes};
}
void AllSources::Identify(
std::ostream &o, Provenance at, const std::string &prefix) const {
static const std::string indented{prefix + " "};
const Origin &origin{MapToOrigin(at)};
std::visit(
visitors{[&](const Inclusion &inc) {
std::pair<int, int> pos{
inc.source.FindOffsetLineAndColumn(at - origin.start)};
o << prefix << "at line " << pos.first << ", column "
<< pos.second << "in the file " << inc.source.path() << '\n';
if (origin.replaces.bytes > 0) {
o << prefix << " that was included\n";
Identify(o, origin.replaces.start, indented);
}
},
[&](const Macro &mac) {
o << prefix << "in the expansion of a macro that was defined\n";
Identify(o, mac.definition.start, indented);
o << prefix << "... and called\n";
Identify(o, origin.replaces.start, indented);
o << prefix << "... and expanded to\n"
<< indented << mac.expansion << '\n';
},
[&](const CompilerInsertion &ins) {
o << prefix << "in text '" << ins.text
<< "' inserted by the compiler\n";
}},
origin.u);
}
const SourceFile *AllSources::GetSourceFile(Provenance at) const {
const Origin &origin{MapToOrigin(at)};
return std::visit(visitors{[](const Inclusion &inc) { return &inc.source; },
[&origin, this](const Macro &mac) {
return GetSourceFile(origin.replaces.start);
},
[](const CompilerInsertion &) {
return static_cast<const SourceFile *>(nullptr);
}},
origin.u);
}
std::string AllSources::GetPath(Provenance at) const {
const SourceFile *source{GetSourceFile(at)};
return source ? source->path() : ""s;
}
int AllSources::GetLineNumber(Provenance at) const {
const SourceFile *source{GetSourceFile(at)};
return source ? source->FindOffsetLineAndColumn(at).first : 0;
}
AllSources::Origin::Origin(size_t s, const SourceFile &source)
: start{s}, u{Inclusion{source}} {}
AllSources::Origin::Origin(
size_t s, const SourceFile &included, ProvenanceRange from)
: start{s}, u{Inclusion{included}}, replaces{from} {}
AllSources::Origin::Origin(size_t s, ProvenanceRange def, ProvenanceRange use,
const std::string &expansion)
: start{s}, u{Macro{def, expansion}}, replaces{use} {}
AllSources::Origin::Origin(size_t s, const std::string &text)
: start{s}, u{CompilerInsertion{text}} {}
size_t AllSources::Origin::size() const {
return std::visit(
visitors{[](const Inclusion &inc) { return inc.source.bytes(); },
[](const Macro &mac) { return mac.expansion.size(); },
[](const CompilerInsertion &ins) { return ins.text.size(); }},
u);
}
const char &AllSources::Origin::operator[](size_t n) const {
return std::visit(
visitors{[n](const Inclusion &inc) -> const char & {
return inc.source.content()[n];
},
[n](const Macro &mac) -> const char & { return mac.expansion[n]; },
[n](const CompilerInsertion &ins) -> const char & {
return ins.text[n];
}},
u);
}
const AllSources::Origin &AllSources::MapToOrigin(Provenance at) const {
CHECK(at < bytes_);
size_t low{0}, count{origin_.size()};
while (count > 1) {
size_t mid{low + (count >> 1)};
if (origin_[mid].start > at) {
count = mid - low;
} else {
count -= mid - low;
low = mid;
}
}
CHECK(at >= origin_[low].start);
CHECK(low + 1 == origin_.size() || at < origin_[low + 1].start);
return origin_[low];
}
ProvenanceRange CookedSource::GetProvenance(const char *at) const {
return provenanceMap_.Map(at - &data_[0]);
}
void CookedSource::Marshal() {
data_.resize(buffer_.size());
char *p{&data_[0]};
for (char ch : buffer_) {
*p++ = ch;
}
buffer_.clear();
}
} // namespace parser
} // namespace Fortran

196
flang/lib/parser/provenance.h
View File

@ -1,38 +1,78 @@
#ifndef FORTRAN_PROVENANCE_H_
#define FORTRAN_PROVENANCE_H_
#include "char-buffer.h"
#include "source.h"
#include <memory>
#include <ostream>
#include <string>
#include <utility>
#include <variant>
#include <vector>
namespace Fortran {
namespace parser {
// Each character in the contiguous source stream built by the
// prescanner corresponds to a particular character in a source file,
// include file, macro expansion, or compiler-inserted padding.
// The location of this original character to which a parsable character
// corresponds is its provenance.
//
// Provenances are offsets into an unmaterialized marshaling of all of the
// entire contents of the original source files, include files, macro
// expansions, &c. for each visit to each source. These origins of the
// original source characters constitute a forest whose roots are
// the original source files named on the compiler's command line.
// We can describe provenances precisely by walking up this tree.
using Provenance = size_t;
class ProvenanceRange {
public:
struct ProvenanceRange {
ProvenanceRange() {}
bool empty() const { return bytes_ == 0; }
Provenance start() const { return start_; }
size_t bytes() const { return bytes_; }
private:
Provenance start_{0};
size_t bytes_{0};
ProvenanceRange(Provenance s, size_t n) : start{s}, bytes{n} {}
ProvenanceRange(const ProvenanceRange &) = default;
ProvenanceRange(ProvenanceRange &&) = default;
ProvenanceRange &operator=(const ProvenanceRange &) = default;
ProvenanceRange &operator=(ProvenanceRange &&) = default;
Provenance start{0};
size_t bytes{0};
};
class AllOfTheSource;
class Origin {
class OffsetToProvenanceMappings {
public:
explicit Origin(const SourceFile &); // initial source file
Origin(const SourceFile &, ProvenanceRange); // included source file
Origin(ProvenanceRange def, ProvenanceRange use, // macro call
const std::string &expansion);
size_t size() const;
const char &operator[](size_t) const;
void Identify(std::ostream &, const AllOfTheSource &, size_t,
const std::string &indent) const;
OffsetToProvenanceMappings() {}
size_t size() const { return bytes_; }
void shrink_to_fit() { provenanceMap_.shrink_to_fit(); }
void Put(ProvenanceRange);
void Put(const OffsetToProvenanceMappings &);
ProvenanceRange Map(size_t at) const;
void RemoveLastBytes(size_t);
private:
struct ContiguousProvenanceMapping {
size_t start;
ProvenanceRange range;
};
size_t bytes_{0};
std::vector<ContiguousProvenanceMapping> provenanceMap_;
};
class AllSources {
public:
explicit AllSources(const SourceFile &initialSourceFile);
size_t size() const { return bytes_; }
const char &operator[](Provenance) const;
ProvenanceRange AddIncludedFile(const SourceFile &, ProvenanceRange);
ProvenanceRange AddMacroCall(
ProvenanceRange def, ProvenanceRange use, const std::string &expansion);
ProvenanceRange AddCompilerInsertion(const std::string &);
void Identify(std::ostream &, Provenance, const std::string &prefix) const;
const SourceFile *GetSourceFile(Provenance) const;
std::string GetPath(Provenance) const; // __FILE__
int GetLineNumber(Provenance) const; // __LINE__
private:
struct Inclusion {
const SourceFile &source;
@ -41,82 +81,58 @@ private:
ProvenanceRange definition;
std::string expansion;
};
std::variant<Inclusion, Macro> u_;
ProvenanceRange replaces_;
};
struct CompilerInsertion {
std::string text;
};
struct Origin {
Origin(size_t start, const SourceFile &);
Origin(size_t start, const SourceFile &, ProvenanceRange);
Origin(size_t start, ProvenanceRange def, ProvenanceRange use,
const std::string &expansion);
Origin(size_t start, const std::string &);
size_t size() const;
const char &operator[](size_t) const;
class AllOfTheSource {
public:
AllOfTheSource() {}
AllOfTheSource(AllOfTheSource &&) = default;
AllOfTheSource &operator=(AllOfTheSource &&) = default;
size_t size() const { return bytes_; }
const char &operator[](Provenance) const;
AllOfTheSource &Add(Origin &&);
void Identify(std::ostream &, Provenance, const std::string &prefix) const;
private:
struct Chunk {
Chunk(Origin &&origin, size_t at) : origin{std::move(origin)}, start{at} {}
Origin origin;
size_t start;
};
const Chunk &MapToChunk(Provenance) const;
std::vector<Chunk> chunk_;
size_t bytes_;
};
class ProvenancedChar {
public:
using type = char;
char character() const { return static_cast<char>(packed_); }
Provenance provenance() const { return packed_ >> 8; }
private:
size_t packed_;
};
class ProvenancedString {
private:
class iterator {
public:
iterator(const AllOfTheSource &sources, Provenance at)
: sources_{&sources}, at_{at} {}
iterator(const iterator &that)
: sources_{that.sources_}, at_{that.at_} {}
iterator &operator=(const iterator &that) {
sources_ = that.sources_;
at_ = that.at_;
return *this;
}
const char &operator*() const;
iterator &operator++() {
++at_;
return *this;
}
iterator operator++(int) {
iterator result{*this};
++at_;
return result;
}
bool operator<(const iterator &that) { return at_ < that.at_; }
bool operator<=(const iterator &that) { return at_ <= that.at_; }
bool operator==(const iterator &that) { return at_ == that.at_; }
bool operator!=(const iterator &that) { return at_ != that.at_; }
private:
const AllOfTheSource *sources_;
size_t at_;
std::variant<Inclusion, Macro, CompilerInsertion> u;
ProvenanceRange replaces;
};
iterator begin(const AllOfTheSource &sources) const {
return iterator(sources, start_);
}
iterator end(const AllOfTheSource &sources) const {
return iterator(sources, start_ + bytes_);
}
const Origin &MapToOrigin(Provenance) const;
std::vector<Origin> origin_;
size_t bytes_{0};
};
class CookedSource {
public:
size_t size() const { return bytes_; }
explicit CookedSource(AllSources &sources) : sources_{sources} {}
size_t size() const { return data_.size(); }
const char &operator[](size_t n) const { return data_[n]; }
const char &at(size_t n) const { return data_.at(n); }
AllSources &sources() const { return sources_; }
ProvenanceRange GetProvenance(const char *) const;
void Identify(std::ostream &, const char *) const;
void Put(const char *data, size_t bytes) { buffer_.Put(data, bytes); }
void Put(char ch) { buffer_.Put(&ch, 1); }
void Put(char ch, Provenance p) {
buffer_.Put(&ch, 1);
provenanceMap_.Put(ProvenanceRange{p, 1});
}
void Put(const OffsetToProvenanceMappings &pm) { provenanceMap_.Put(pm); }
void Marshal(); // marshalls all text into one contiguous block
private:
Provenance start_;
size_t bytes_;
AllSources &sources_;
CharBuffer buffer_; // before Marshal()
std::vector<char> data_; // all of it, prescanned and preprocessed
OffsetToProvenanceMappings provenanceMap_;
};
} // namespace parser
} // namespace Fortran

16
flang/lib/parser/source.cc
View File

@ -105,7 +105,7 @@ bool SourceFile::Open(std::string path, std::stringstream *error) {
}
close(fileDescriptor_);
fileDescriptor_ = -1;
bytes_ = buffer.bytes();
bytes_ = buffer.size();
if (bytes_ == 0) {
// empty file
content_ = nullptr;
@ -144,7 +144,7 @@ void SourceFile::Close() {
path_.clear();
}
Position SourceFile::FindOffsetPosition(size_t at) const {
std::pair<int, int> SourceFile::FindOffsetLineAndColumn(size_t at) const {
CHECK(at < bytes_);
if (lineStart_.empty()) {
return {1, static_cast<int>(at + 1)};
@ -159,16 +159,8 @@ Position SourceFile::FindOffsetPosition(size_t at) const {
low = mid;
}
}
return {static_cast<int>(low + 1),
static_cast<int>(at - lineStart_[low] + 1)};
}
size_t SourceFile::FindPositionOffset(int lineNumber, int column) const {
return lineStart_.at(lineNumber - 1) + column - 1;
}
size_t SourceFile::FindPositionOffset(Position pos) const {
return FindPositionOffset(pos.lineNumber(), pos.column());
return {
static_cast<int>(low + 1), static_cast<int>(at - lineStart_[low] + 1)};
}
} // namespace parser
} // namespace Fortran

6
flang/lib/parser/source.h
View File

@ -5,9 +5,9 @@
// - Line ending markers are converted to single newline characters
// - A newline character is added to the last line of the file if one is needed
#include "position.h"
#include <sstream>
#include <string>
#include <utility>
#include <vector>
namespace Fortran {
@ -23,9 +23,7 @@ public:
const char *content() const { return content_; }
size_t bytes() const { return bytes_; }
size_t lines() const { return lineStart_.size(); }
Position FindOffsetPosition(size_t) const;
size_t FindPositionOffset(int lineNumber, int column) const;
size_t FindPositionOffset(Position) const;
std::pair<int, int> FindOffsetLineAndColumn(size_t) const;
private:
std::string path_;

50
flang/tools/f18/f18.cc
View File

@ -8,6 +8,7 @@
#include "../../lib/parser/message.h"
#include "../../lib/parser/parse-tree.h"
#include "../../lib/parser/prescan.h"
#include "../../lib/parser/provenance.h"
#include "../../lib/parser/source.h"
#include "../../lib/parser/user-state.h"
#include <cerrno>
@ -54,7 +55,6 @@ int main(int argc, char *const argv[]) {
bool standard{false};
bool enableOldDebugLines{false};
int columns{72};
bool prescan{true};
while (!args.empty()) {
if (args.front().empty()) {
@ -77,8 +77,6 @@ int main(int argc, char *const argv[]) {
columns = 132;
} else if (flag == "-fdebug-dump-cooked-chars") {
dumpCookedChars = true;
} else if (flag == "-fno-prescan") {
prescan = false;
} else if (flag == "-ed") {
enableOldDebugLines = true;
} else {
@ -99,39 +97,29 @@ int main(int argc, char *const argv[]) {
}
}
Fortran::parser::SourceFile source;
Fortran::parser::SourceFile sourceFile;
std::stringstream error;
if (!source.Open(path, &error)) {
if (!sourceFile.Open(path, &error)) {
std::cerr << error.str() << '\n';
return 1;
}
const char *sourceContent{source.content()};
size_t sourceBytes{source.bytes()};
std::unique_ptr<char[]> prescanned;
if (prescan) {
Fortran::parser::Messages messages;
Fortran::parser::Prescanner prescanner{messages};
Fortran::parser::CharBuffer buffer{
prescanner.set_fixedForm(fixedForm)
.set_enableBackslashEscapesInCharLiterals(backslashEscapes)
.set_fixedFormColumnLimit(columns)
.set_enableOldDebugLines(enableOldDebugLines)
.Prescan(source)};
std::cerr << messages;
if (prescanner.anyFatalErrors()) {
return 1;
}
sourceBytes = buffer.bytes();
char *contig{new char[sourceBytes]};
buffer.CopyToContiguous(contig);
sourceContent = contig;
prescanned.reset(contig);
columns = std::numeric_limits<int>::max();
Fortran::parser::AllSources allSources{sourceFile};
Fortran::parser::Messages messages;
Fortran::parser::Prescanner prescanner{messages, allSources};
Fortran::parser::CookedSource cooked{
prescanner.set_fixedForm(fixedForm)
.set_enableBackslashEscapesInCharLiterals(backslashEscapes)
.set_fixedFormColumnLimit(columns)
.set_enableOldDebugLines(enableOldDebugLines)
.Prescan()};
messages.Emit(std::cerr, allSources);
if (prescanner.anyFatalErrors()) {
return 1;
}
columns = std::numeric_limits<int>::max();
Fortran::parser::ParseState state{sourceContent, sourceBytes};
state.set_prescanned(prescan);
Fortran::parser::ParseState state{cooked};
state.set_inFixedForm(fixedForm);
state.set_enableBackslashEscapesInCharLiterals(backslashEscapes);
state.set_strictConformance(standard);
@ -164,7 +152,9 @@ int main(int argc, char *const argv[]) {
if (result.has_value() && !state.anyErrorRecovery()) {
std::cout << "demo PASS\n" << *result << '\n';
} else {
std::cerr << "demo FAIL " << state.position() << '\n' << *state.messages();
std::cerr << "demo FAIL\n";
allSources.Identify(std::cerr, state.GetProvenance(), " ");
state.messages()->Emit(std::cerr, allSources);
return EXIT_FAILURE;
}
}