[flang] Pass as much of f90_correct as I think we can.

Original-commit: flang-compiler/f18@6d101d3ec8
Reviewed-on: https://github.com/flang-compiler/f18/pull/26
Tree-same-pre-rewrite: false

flang/lib/parser/preprocessor.cc

@@ -564,12 +564,12 @@ static std::string GetDirectiveName(const TokenSequence &line, size_t *rest) {
   size_t j{SkipBlanks(line, 0, tokens)};
   if (j == tokens || line[j].ToString() != "#") {
     *rest = tokens;
-    return {};
+    return "";
   }
   j = SkipBlanks(line, j + 1, tokens);
   if (j == tokens) {
     *rest = tokens;
-    return {};
+    return "";
   }
   *rest = SkipBlanks(line, j + 1, tokens);
   return ToLowerCaseLetters(line[j].ToString());
@@ -578,10 +578,14 @@ static std::string GetDirectiveName(const TokenSequence &line, size_t *rest) {
 void Preprocessor::SkipDisabledConditionalCode(const std::string &dirName,
     IsElseActive isElseActive, Prescanner *prescanner) {
   int nesting{0};
-  while (
-      std::optional<TokenSequence> line{prescanner->NextTokenizedLine(false)}) {
+  while (!prescanner->IsAtEnd()) {
+    if (!prescanner->IsNextLinePreprocessorDirective()) {
+      prescanner->NextLine();
+      continue;
+    }
+    TokenSequence line{prescanner->TokenizePreprocessorDirective()};
     size_t rest{0};
-    std::string dn{GetDirectiveName(*line, &rest)};
+    std::string dn{GetDirectiveName(line, &rest)};
     if (dn == "ifdef" || dn == "ifndef" || dn == "if") {
       ++nesting;
     } else if (dn == "endif") {
@@ -594,7 +598,7 @@ void Preprocessor::SkipDisabledConditionalCode(const std::string &dirName,
         return;
       }
       if (dn == "elif" &&
-          IsIfPredicateTrue(*line, rest, line->size() - rest, prescanner)) {
+          IsIfPredicateTrue(line, rest, line.size() - rest, prescanner)) {
         ifStack_.push(CanDeadElseAppear::Yes);
         return;
       }

flang/lib/parser/prescan.cc

@@ -69,14 +69,10 @@ bool Prescanner::Prescan(ProvenanceRange range) {
   return !anyFatalErrors_;
 }
 
-std::optional<TokenSequence> Prescanner::NextTokenizedLine(
-    bool isPreprocessorDirective) {
-  if (lineStart_ >= limit_) {
-    return {};
-  }
+TokenSequence Prescanner::TokenizePreprocessorDirective() {
+  CHECK(lineStart_ < limit_ && !inPreprocessorDirective_);
   auto saveAt = at_;
-  inPreprocessorDirective_ =
-      isPreprocessorDirective || IsPreprocessorDirectiveLine(lineStart_);
+  inPreprocessorDirective_ = true;
   BeginSourceLineAndAdvance();
   TokenSequence tokens;
   while (NextToken(&tokens)) {
@@ -321,9 +317,11 @@ void Prescanner::QuotedCharacterLiteral(TokenSequence *tokens) {
     while (PadOutCharacterLiteral(tokens)) {
     }
     if (*at_ == '\n') {
+      if (!inPreprocessorDirective_) {
         messages_->Put(
             {GetProvenance(start), "incomplete character literal"_en_US});
         anyFatalErrors_ = true;
+      }
       break;
     }
     NextChar();
@@ -371,9 +369,11 @@ void Prescanner::Hollerith(TokenSequence *tokens, int count) {
     }
   }
   if (*at_ == '\n') {
+    if (!inPreprocessorDirective_) {
       messages_->Put(
           {GetProvenance(start), "incomplete Hollerith literal"_en_US});
       anyFatalErrors_ = true;
+    }
   } else {
     NextChar();
   }
@@ -507,7 +507,7 @@ bool Prescanner::IncludeLine(const char *p) {
   return true;
 }
 
-bool Prescanner::IsPreprocessorDirectiveLine(const char *start) {
+bool Prescanner::IsPreprocessorDirectiveLine(const char *start) const {
   const char *p{start};
   if (p >= limit_) {
     return false;
@@ -522,6 +522,10 @@ bool Prescanner::IsPreprocessorDirectiveLine(const char *start) {
   return *p == '#';
 }
 
+bool Prescanner::IsNextLinePreprocessorDirective() const {
+  return IsPreprocessorDirectiveLine(lineStart_);
+}
+
 bool Prescanner::CommentLines() {
   bool any{false};
   while (lineStart_ < limit_) {
@@ -542,7 +546,8 @@ bool Prescanner::CommentLinesAndPreprocessorDirectives() {
         IncludeLine(lineStart_)) {
       NextLine();
     } else if (IsPreprocessorDirectiveLine(lineStart_)) {
-      if (std::optional<TokenSequence> tokens{NextTokenizedLine(true)}) {
+      if (std::optional<TokenSequence> tokens{
+              TokenizePreprocessorDirective()}) {
         preprocessor_->Directive(*tokens, this);
       }
     } else {

flang/lib/parser/prescan.h

@@ -52,9 +52,12 @@ public:
   }
 
   bool Prescan(ProvenanceRange);
+  void NextLine();
 
   // Callbacks for use by Preprocessor.
-  std::optional<TokenSequence> NextTokenizedLine(bool isPreprocessorDirective);
+  bool IsAtEnd() const { return lineStart_ >= limit_; }
+  bool IsNextLinePreprocessorDirective() const;
+  TokenSequence TokenizePreprocessorDirective();
   Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
   Message &Error(MessageFixedText);
   Message &Error(MessageFormattedText &&);
@@ -95,7 +98,6 @@ private:
     return *at_;
   }
 
-  void NextLine();
   void LabelField(TokenSequence *);
   void NextChar();
   void SkipSpaces();
@@ -109,7 +111,7 @@ private:
   bool IsFixedFormCommentLine(const char *);
   bool IsFreeFormComment(const char *);
   bool IncludeLine(const char *);
-  bool IsPreprocessorDirectiveLine(const char *);
+  bool IsPreprocessorDirectiveLine(const char *) const;
   const char *FixedFormContinuationLine();
   bool FixedFormContinuation();
   bool FreeFormContinuation();