[flang] Implement Fortran INCLUDE lines.

Original-commit: flang-compiler/f18@d01af89506
Reviewed-on: https://github.com/flang-compiler/f18/pull/9
Tree-same-pre-rewrite: false

flang/lib/parser/prescan.cc

@@ -6,6 +6,7 @@
 #include "token-sequence.h"
 #include <cctype>
 #include <cstring>
+#include <sstream>
 #include <utility>
 #include <vector>
 
@@ -16,6 +17,14 @@ Prescanner::Prescanner(
     Messages *messages, CookedSource *cooked, Preprocessor *preprocessor)
   : messages_{messages}, cooked_{cooked}, preprocessor_{preprocessor} {}
 
+Prescanner::Prescanner(const Prescanner &that)
+  : messages_{that.messages_}, cooked_{that.cooked_},
+    preprocessor_{that.preprocessor_}, inFixedForm_{that.inFixedForm_},
+    fixedFormColumnLimit_{that.fixedFormColumnLimit_},
+    enableOldDebugLines_{that.enableOldDebugLines_},
+    enableBackslashEscapesInCharLiterals_{
+        that.enableBackslashEscapesInCharLiterals_} {}
+
 bool Prescanner::Prescan(ProvenanceRange range) {
   startProvenance_ = range.start;
   ProvenanceRange around{
@@ -30,6 +39,7 @@ bool Prescanner::Prescan(ProvenanceRange range) {
   TokenSequence tokens, preprocessed;
   while (lineStart_ < limit_) {
     if (CommentLinesAndPreprocessorDirectives() && lineStart_ >= limit_) {
+      PayNewlineDebt();
       break;
     }
     BeginSourceLineAndAdvance();
@@ -56,9 +66,9 @@ bool Prescanner::Prescan(ProvenanceRange range) {
     }
     tokens.clear();
     cooked_->Put('\n', newlineProvenance_);
-    PayNewlineDebt(cooked_);
+    PayNewlineDebt();
   }
-  PayNewlineDebt(cooked_);
+  PayNewlineDebt();
   return !anyFatalErrors_;
 }
 
@@ -373,6 +383,62 @@ bool Prescanner::IsFreeFormComment(const char *p) {
   return *p == '!' || *p == '\n';
 }
 
+bool Prescanner::IncludeLine(const char *p) {
+  if (p >= limit_) {
+    return false;
+  }
+  const char *start{p};
+  while (*p == ' ' || *p == '\t') {
+    ++p;
+  }
+  for (char ch : "include"s) {
+    if (tolower(*p++) != ch) {
+      return false;
+    }
+  }
+  while (*p == ' ' || *p == '\t') {
+    ++p;
+  }
+  if (*p != '"' && *p != '\'') {
+    return false;
+  }
+  char quote{*p};
+  std::string path;
+  for (++p; *p != '\n'; ++p) {
+    if (*p == quote) {
+      if (p[1] != quote) {
+        break;
+      }
+      ++p;
+    }
+    path += *p;
+  }
+  if (*p != quote) {
+    messages_->Put({GetProvenance(p), "malformed path name string"});
+    anyFatalErrors_ = true;
+    return true;
+  }
+  for (++p; *p == ' ' || *p == '\t'; ++p) {
+  }
+  if (*p != '\n' && *p != '!') {
+    messages_->Put({GetProvenance(p), "excess characters after path name"});
+  }
+  std::stringstream error;
+  Provenance provenance{GetProvenance(start)};
+  AllSources *allSources{cooked_->allSources()};
+  const SourceFile *included{allSources->Open(path, &error)};
+  if (included == nullptr) {
+    messages_->Put({provenance, error.str()});
+    anyFatalErrors_ = true;
+    return true;
+  }
+  ProvenanceRange includeLineRange{provenance, static_cast<size_t>(p - start)};
+  ProvenanceRange fileRange{
+      allSources->AddIncludedFile(*included, includeLineRange)};
+  anyFatalErrors_ |= !Prescanner{*this}.Prescan(fileRange);
+  return true;
+}
+
 bool Prescanner::IsPreprocessorDirectiveLine(const char *start) {
   const char *p{start};
   if (p >= limit_ || inPreprocessorDirective_) {
@@ -405,7 +471,8 @@ bool Prescanner::CommentLines() {
 bool Prescanner::CommentLinesAndPreprocessorDirectives() {
   bool any{false};
   while (lineStart_ < limit_) {
-    if (IsFixedFormCommentLine(lineStart_) || IsFreeFormComment(lineStart_)) {
+    if (IsFixedFormCommentLine(lineStart_) || IsFreeFormComment(lineStart_) ||
+        IncludeLine(lineStart_)) {
       NextLine();
     } else if (IsPreprocessorDirectiveLine(lineStart_)) {
       if (std::optional<TokenSequence> tokens{NextTokenizedLine()}) {
@@ -499,9 +566,9 @@ bool Prescanner::FreeFormContinuation() {
   return true;
 }
 
-void Prescanner::PayNewlineDebt(CookedSource *cooked) {
+void Prescanner::PayNewlineDebt() {
   for (; newlineDebt_ > 0; --newlineDebt_) {
-    cooked->Put('\n', newlineProvenance_);
+    cooked_->Put('\n', newlineProvenance_);
   }
 }
 }  // namespace parser

flang/lib/parser/prescan.h

@@ -22,6 +22,7 @@ class Preprocessor;
 class Prescanner {
 public:
   Prescanner(Messages *, CookedSource *, Preprocessor *);
+  Prescanner(const Prescanner &);
 
   Messages *messages() const { return messages_; }
 
@@ -42,9 +43,9 @@ public:
     return *this;
   }
 
-  CookedSource *cooked() const { return cooked_; }
-
   bool Prescan(ProvenanceRange);
+
+  // Callbacks for use by Preprocessor.
   std::optional<TokenSequence> NextTokenizedLine();
   Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
   void Complain(const std::string &message);
@@ -89,11 +90,12 @@ private:
   bool CommentLinesAndPreprocessorDirectives();
   bool IsFixedFormCommentLine(const char *);
   bool IsFreeFormComment(const char *);
+  bool IncludeLine(const char *);
   bool IsPreprocessorDirectiveLine(const char *);
   const char *FixedFormContinuationLine();
   bool FixedFormContinuation();
   bool FreeFormContinuation();
-  void PayNewlineDebt(CookedSource *);
+  void PayNewlineDebt();
 
   Messages *messages_;
   CookedSource *cooked_;

flang/lib/parser/provenance.cc

@@ -68,11 +68,21 @@ AllSources::AllSources() {
   }
 }
 
+AllSources::~AllSources() {}
+
 const char &AllSources::operator[](Provenance at) const {
   const Origin &origin{MapToOrigin(at)};
   return origin[at - origin.start];
 }
 
+const SourceFile *AllSources::Open(std::string path, std::stringstream *error) {
+  std::unique_ptr<SourceFile> source{std::make_unique<SourceFile>()};
+  if (source->Open(path, error)) {
+    return ownedSourceFiles_.emplace_back(std::move(source)).get();
+  }
+  return nullptr;
+}
+
 ProvenanceRange AllSources::AddIncludedFile(
     const SourceFile &source, ProvenanceRange from) {
   size_t start{bytes_}, bytes{source.bytes()};

flang/lib/parser/provenance.h

@@ -1,13 +1,17 @@
 #ifndef FORTRAN_PROVENANCE_H_
 #define FORTRAN_PROVENANCE_H_
+
 #include "char-buffer.h"
 #include "source.h"
 #include <map>
+#include <memory>
 #include <ostream>
+#include <sstream>
 #include <string>
 #include <utility>
 #include <variant>
 #include <vector>
+
 namespace Fortran {
 namespace parser {
 
@@ -66,10 +70,13 @@ private:
 class AllSources {
 public:
   AllSources();
+  ~AllSources();
 
   size_t size() const { return bytes_; }
   const char &operator[](Provenance) const;
 
+  const SourceFile *Open(std::string path, std::stringstream *error);
+
   ProvenanceRange AddIncludedFile(const SourceFile &, ProvenanceRange);
   ProvenanceRange AddMacroCall(
       ProvenanceRange def, ProvenanceRange use, const std::string &expansion);
@@ -114,6 +121,7 @@ private:
   std::vector<Origin> origin_;
   size_t bytes_{0};
   std::map<char, Provenance> compilerInsertionProvenance_;
+  std::vector<std::unique_ptr<SourceFile>> ownedSourceFiles_;
 };
 
 class CookedSource {

flang/tools/f18/f18.cc

@@ -10,7 +10,6 @@
 #include "../../lib/parser/preprocessor.h"
 #include "../../lib/parser/prescan.h"
 #include "../../lib/parser/provenance.h"
-#include "../../lib/parser/source.h"
 #include "../../lib/parser/user-state.h"
 #include <cerrno>
 #include <cstdio>
@@ -98,16 +97,16 @@ int main(int argc, char *const argv[]) {
     }
   }
 
-  Fortran::parser::SourceFile sourceFile;
+  Fortran::parser::AllSources allSources;
   std::stringstream error;
-  if (!sourceFile.Open(path, &error)) {
+  const auto *sourceFile = allSources.Open(path, &error);
+  if (!sourceFile) {
     std::cerr << error.str() << '\n';
     return 1;
   }
 
-  Fortran::parser::AllSources allSources;
   Fortran::parser::ProvenanceRange range{allSources.AddIncludedFile(
-      sourceFile, Fortran::parser::ProvenanceRange{})};
+      *sourceFile, Fortran::parser::ProvenanceRange{})};
   Fortran::parser::Messages messages{allSources};
   Fortran::parser::CookedSource cooked{&allSources};
   Fortran::parser::Preprocessor preprocessor{&allSources};