[flang] Quash multiple blanks when preprocessing.

Original-commit: flang-compiler/f18@d91680b469
Reviewed-on: https://github.com/flang-compiler/f18/pull/87
Tree-same-pre-rewrite: false

flang/lib/parser/preprocessor.cc

@@ -344,6 +344,9 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
     }
     j = k;  // advance to the terminal ')'
   }
+  if (result.HasRedundantBlanks()) {
+    result.RemoveRedundantBlanks();
+  }
   return {result};
 }
 

flang/lib/parser/prescan.cc

@@ -89,7 +89,7 @@ void Prescanner::Prescan(ProvenanceRange range) {
     }
     dir += '\n';
     TokenSequence tokens{dir, allSources.AddCompilerInsertion(dir).start()};
-    tokens.Emit(&cooked_);
+    tokens.Emit(cooked_);
   }
 }
 
@@ -160,16 +160,16 @@ void Prescanner::Statement() {
     case LineClassification::Kind::PreprocessorDirective:
       Say("preprocessed line resembles a preprocessor directive"_en_US,
           preprocessed->GetProvenanceRange());
-      preprocessed->ToLowerCase().Emit(&cooked_);
+      preprocessed->ToLowerCase().Emit(cooked_);
       break;
     case LineClassification::Kind::CompilerDirective:
       NormalizeCompilerDirectiveCommentMarker(*preprocessed);
       preprocessed->ToLowerCase();
       SourceFormChange(preprocessed->ToString());
-      preprocessed->Emit(&cooked_);
+      preprocessed->Emit(cooked_);
       break;
     case LineClassification::Kind::Source:
-      preprocessed->ToLowerCase().Emit(&cooked_);
+      preprocessed->ToLowerCase().Emit(cooked_);
       break;
     }
   } else {
@@ -177,7 +177,7 @@ void Prescanner::Statement() {
     if (line.kind == LineClassification::Kind::CompilerDirective) {
       SourceFormChange(tokens.ToString());
     }
-    tokens.Emit(&cooked_);
+    tokens.Emit(cooked_);
     cooked_.Put('\n', newlineProvenance);
   }
   directiveSentinel_ = nullptr;

flang/lib/parser/token-sequence.cc

@@ -144,9 +144,47 @@ TokenSequence &TokenSequence::ToLowerCase() {
   return *this;
 }
 
-void TokenSequence::Emit(CookedSource *cooked) const {
-  cooked->Put(&char_[0], char_.size());
-  cooked->PutProvenanceMappings(provenances_);
+bool TokenSequence::HasRedundantBlanks() const {
+  std::size_t tokens{SizeInTokens()};
+  bool lastWasBlank{false};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    bool isBlank{TokenAt(j).IsBlank()};
+    if (isBlank && lastWasBlank) {
+      return true;
+    }
+    lastWasBlank = isBlank;
+  }
+  return false;
+}
+
+TokenSequence &TokenSequence::RemoveRedundantBlanks() {
+  std::size_t tokens{SizeInTokens()};
+  TokenSequence result;
+  bool lastWasBlank{false};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    bool isBlank{TokenAt(j).IsBlank()};
+    if (isBlank && lastWasBlank) {
+      continue;
+    }
+    lastWasBlank = isBlank;
+    result.Put(*this, j);
+  }
+  *this = std::move(result);
+  return *this;
+}
+
+void TokenSequence::Emit(CookedSource &cooked) const {
+  cooked.Put(&char_[0], char_.size());
+  cooked.PutProvenanceMappings(provenances_);
+}
+
+void TokenSequence::Dump(std::ostream &o) const {
+  o << "TokenSequence has " << char_.size() << " chars; nextStart_ "
+    << nextStart_ << '\n';
+  for (std::size_t j{0}; j < start_.size(); ++j) {
+    o << '[' << j << "] @ " << start_[j] << " '" << TokenAt(j).ToString()
+      << "'\n";
+  }
 }
 
 Provenance TokenSequence::GetTokenProvenance(
@@ -176,5 +214,4 @@ ProvenanceRange TokenSequence::GetIntervalProvenanceRange(
 ProvenanceRange TokenSequence::GetProvenanceRange() const {
   return GetIntervalProvenanceRange(0, start_.size());
 }
-
 }  // namespace Fortran::parser

flang/lib/parser/token-sequence.h

@@ -23,6 +23,7 @@
 #include "provenance.h"
 #include <cstddef>
 #include <cstring>
+#include <ostream>
 #include <string>
 #include <utility>
 #include <vector>
@@ -106,7 +107,10 @@ public:
 
   char *GetMutableCharData() { return &char_[0]; }
   TokenSequence &ToLowerCase();
-  void Emit(CookedSource *) const;
+  bool HasRedundantBlanks() const;
+  TokenSequence &RemoveRedundantBlanks();
+  void Emit(CookedSource &) const;
+  void Dump(std::ostream &) const;
 
 private:
   std::size_t TokenBytes(std::size_t token) const {