From f1840f1601b015335960d7a5fccfb0997060d28c Mon Sep 17 00:00:00 2001
From: peter klausler <pklausler@nvidia.com>
Date: Fri, 11 May 2018 11:15:20 -0700
Subject: [PATCH] [flang] Quash multiple blanks when preprocessing.

Original-commit: flang-compiler/f18@d91680b469fc627a1325fe28c7382fd16a279892
Reviewed-on: https://github.com/flang-compiler/f18/pull/87
Tree-same-pre-rewrite: false
---
 flang/lib/parser/preprocessor.cc   |  3 ++
 flang/lib/parser/prescan.cc        | 10 +++----
 flang/lib/parser/token-sequence.cc | 45 +++++++++++++++++++++++++++---
 flang/lib/parser/token-sequence.h  |  6 +++-
 4 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/flang/lib/parser/preprocessor.cc b/flang/lib/parser/preprocessor.cc
index f2b8fc25134f..38fd22d451d7 100644
--- a/flang/lib/parser/preprocessor.cc
+++ b/flang/lib/parser/preprocessor.cc
@@ -344,6 +344,9 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
     }
     j = k;  // advance to the terminal ')'
   }
+  if (result.HasRedundantBlanks()) {
+    result.RemoveRedundantBlanks();
+  }
   return {result};
 }
 
diff --git a/flang/lib/parser/prescan.cc b/flang/lib/parser/prescan.cc
index 8131bf0be425..b3a701fa2a37 100644
--- a/flang/lib/parser/prescan.cc
+++ b/flang/lib/parser/prescan.cc
@@ -89,7 +89,7 @@ void Prescanner::Prescan(ProvenanceRange range) {
     }
     dir += '\n';
     TokenSequence tokens{dir, allSources.AddCompilerInsertion(dir).start()};
-    tokens.Emit(&cooked_);
+    tokens.Emit(cooked_);
   }
 }
 
@@ -160,16 +160,16 @@ void Prescanner::Statement() {
     case LineClassification::Kind::PreprocessorDirective:
       Say("preprocessed line resembles a preprocessor directive"_en_US,
           preprocessed->GetProvenanceRange());
-      preprocessed->ToLowerCase().Emit(&cooked_);
+      preprocessed->ToLowerCase().Emit(cooked_);
       break;
     case LineClassification::Kind::CompilerDirective:
       NormalizeCompilerDirectiveCommentMarker(*preprocessed);
       preprocessed->ToLowerCase();
       SourceFormChange(preprocessed->ToString());
-      preprocessed->Emit(&cooked_);
+      preprocessed->Emit(cooked_);
       break;
     case LineClassification::Kind::Source:
-      preprocessed->ToLowerCase().Emit(&cooked_);
+      preprocessed->ToLowerCase().Emit(cooked_);
       break;
     }
   } else {
@@ -177,7 +177,7 @@ void Prescanner::Statement() {
     if (line.kind == LineClassification::Kind::CompilerDirective) {
       SourceFormChange(tokens.ToString());
     }
-    tokens.Emit(&cooked_);
+    tokens.Emit(cooked_);
     cooked_.Put('\n', newlineProvenance);
   }
   directiveSentinel_ = nullptr;
diff --git a/flang/lib/parser/token-sequence.cc b/flang/lib/parser/token-sequence.cc
index 46787ade5703..5836dd8e948d 100644
--- a/flang/lib/parser/token-sequence.cc
+++ b/flang/lib/parser/token-sequence.cc
@@ -144,9 +144,47 @@ TokenSequence &TokenSequence::ToLowerCase() {
   return *this;
 }
 
-void TokenSequence::Emit(CookedSource *cooked) const {
-  cooked->Put(&char_[0], char_.size());
-  cooked->PutProvenanceMappings(provenances_);
+bool TokenSequence::HasRedundantBlanks() const {
+  std::size_t tokens{SizeInTokens()};
+  bool lastWasBlank{false};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    bool isBlank{TokenAt(j).IsBlank()};
+    if (isBlank && lastWasBlank) {
+      return true;
+    }
+    lastWasBlank = isBlank;
+  }
+  return false;
+}
+
+TokenSequence &TokenSequence::RemoveRedundantBlanks() {
+  std::size_t tokens{SizeInTokens()};
+  TokenSequence result;
+  bool lastWasBlank{false};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    bool isBlank{TokenAt(j).IsBlank()};
+    if (isBlank && lastWasBlank) {
+      continue;
+    }
+    lastWasBlank = isBlank;
+    result.Put(*this, j);
+  }
+  *this = std::move(result);
+  return *this;
+}
+
+void TokenSequence::Emit(CookedSource &cooked) const {
+  cooked.Put(&char_[0], char_.size());
+  cooked.PutProvenanceMappings(provenances_);
+}
+
+void TokenSequence::Dump(std::ostream &o) const {
+  o << "TokenSequence has " << char_.size() << " chars; nextStart_ "
+    << nextStart_ << '\n';
+  for (std::size_t j{0}; j < start_.size(); ++j) {
+    o << '[' << j << "] @ " << start_[j] << " '" << TokenAt(j).ToString()
+      << "'\n";
+  }
 }
 
 Provenance TokenSequence::GetTokenProvenance(
@@ -176,5 +214,4 @@ ProvenanceRange TokenSequence::GetIntervalProvenanceRange(
 ProvenanceRange TokenSequence::GetProvenanceRange() const {
   return GetIntervalProvenanceRange(0, start_.size());
 }
-
 }  // namespace Fortran::parser
diff --git a/flang/lib/parser/token-sequence.h b/flang/lib/parser/token-sequence.h
index ad257c321978..f4d096590afd 100644
--- a/flang/lib/parser/token-sequence.h
+++ b/flang/lib/parser/token-sequence.h
@@ -23,6 +23,7 @@
 #include "provenance.h"
 #include <cstddef>
 #include <cstring>
+#include <ostream>
 #include <string>
 #include <utility>
 #include <vector>
@@ -106,7 +107,10 @@ public:
 
   char *GetMutableCharData() { return &char_[0]; }
   TokenSequence &ToLowerCase();
-  void Emit(CookedSource *) const;
+  bool HasRedundantBlanks() const;
+  TokenSequence &RemoveRedundantBlanks();
+  void Emit(CookedSource &) const;
+  void Dump(std::ostream &) const;
 
 private:
   std::size_t TokenBytes(std::size_t token) const {