[flang] Support unparse with symbol annotations.

When `-fdebug-dump-symbols` is supplied with `-funparse`, include symbol
information in comments in the Fortran output. This will be used for
testing to verify that correct symbols are defined and references in
the right place.

In `UnparseWithSymbols()`, walk the parse tree and collect symbol
definitions and references, organized by statement. When a symbol is
defined across several statement it is associated with the first.
The definition of implicitly defined symbols is associated with the
first reference.

To write out the symbol information, a new optional argument is added to
`Unparse()`: it is a function that is called immediately before each
statement is unparsed. We pass in a function that prints out the symbol
information collected for that statement.

Add `Symbol::GetType()` to make it easier to write the symbol types
and add `Symbol::SetType()` for uniformity.

Original-commit: flang-compiler/f18@2e827de4ad
Reviewed-on: https://github.com/flang-compiler/f18/pull/112
Tree-same-pre-rewrite: false

flang/lib/parser/unparse.cc

@@ -31,9 +31,9 @@ namespace Fortran::parser {
 class UnparseVisitor {
 public:
   UnparseVisitor(std::ostream &out, int indentationAmount, Encoding encoding,
-      bool capitalize)
+      bool capitalize, preStatementType *preStatement)
     : out_{out}, indentationAmount_{indentationAmount}, encoding_{encoding},
-      capitalizeKeywords_{capitalize} {}
+      capitalizeKeywords_{capitalize}, preStatement_{preStatement} {}
 
   // In nearly all cases, this code avoids defining Boolean-valued Pre()
   // callbacks for the parse tree walking framework in favor of two void
@@ -66,6 +66,9 @@ public:
 
   // Statement labels and ends of lines
   template<typename T> void Before(const Statement<T> &x) {
+    if (preStatement_) {
+      (*preStatement_)(x.source, out_, indent_);
+    }
     Walk(x.label, " ");
   }
   template<typename T> void Post(const Statement<T> &) { Put('\n'); }
@@ -2201,6 +2204,7 @@ private:
   std::set<CharBlock> structureComponents_;
   Encoding encoding_{Encoding::UTF8};
   bool capitalizeKeywords_{true};
+  preStatementType *preStatement_{nullptr};
 };
 
 void UnparseVisitor::Put(char ch) {
@@ -2263,8 +2267,8 @@ void UnparseVisitor::Word(const char *str) {
 void UnparseVisitor::Word(const std::string &str) { Word(str.c_str()); }
 
 void Unparse(std::ostream &out, const Program &program, Encoding encoding,
-    bool capitalizeKeywords) {
-  UnparseVisitor visitor{out, 1, encoding, capitalizeKeywords};
+    bool capitalizeKeywords, preStatementType *preStatement) {
+  UnparseVisitor visitor{out, 1, encoding, capitalizeKeywords, preStatement};
   Walk(program, visitor);
   visitor.Done();
 }

flang/lib/parser/unparse.h

@@ -15,16 +15,23 @@
 #ifndef FORTRAN_PARSER_UNPARSE_H_
 #define FORTRAN_PARSER_UNPARSE_H_
 
+#include "char-block.h"
 #include "characters.h"
+#include <functional>
 #include <iosfwd>
 
 namespace Fortran::parser {
 
 struct Program;
 
+// A function called before each Statement is unparsed.
+using preStatementType =
+    std::function<void(const CharBlock &, std::ostream &, int)>;
+
 /// Convert parsed program to out as Fortran.
 void Unparse(std::ostream &out, const Program &program,
-    Encoding encoding = Encoding::UTF8, bool capitalizeKeywords = true);
+    Encoding encoding = Encoding::UTF8, bool capitalizeKeywords = true,
+    preStatementType *preStatement = nullptr);
 
 }  // namespace Fortran::parser
 

flang/lib/semantics/CMakeLists.txt

@@ -20,6 +20,7 @@ add_library(FlangSemantics
   scope.cc
   symbol.cc
   type.cc
+  unparse-with-symbols.cc
 )
 
 target_link_libraries(FlangSemantics

flang/lib/semantics/resolve-names.cc

@@ -1798,6 +1798,7 @@ bool DeclarationVisitor::HandleAttributeStmt(
             EnumToString(attr), name.source);
       }
       symbol.attrs().set(attr);
+      symbol.add_occurrence(name.source);
     }
   }
   return false;
@@ -1959,25 +1960,11 @@ bool DeclarationVisitor::Pre(const parser::FinalProcedureStmt &x) {
 
 void DeclarationVisitor::SetType(
     const SourceName &name, Symbol &symbol, const DeclTypeSpec &type) {
-  if (auto *details = symbol.detailsIf<EntityDetails>()) {
-    if (!details->type()) {
-      details->set_type(type);
-      return;
-    }
-  } else if (auto *details = symbol.detailsIf<ObjectEntityDetails>()) {
-    if (!details->type()) {
-      details->set_type(type);
-      return;
-    }
-  } else if (auto *details = symbol.detailsIf<ProcEntityDetails>()) {
-    if (!details->interface().type()) {
-      details->interface().set_type(type);
-      return;
-    }
-  } else {
+  if (symbol.GetType()) {
+    Say(name, "The type of '%s' has already been declared"_err_en_US);
     return;
   }
-  Say(name, "The type of '%s' has already been declared"_err_en_US);
+  symbol.SetType(type);
 }
 
 // ResolveNamesVisitor implementation
@@ -2115,7 +2102,9 @@ const Symbol *ResolveNamesVisitor::FindComponent(
                 "Declaration of '%s'"_en_US, typeName.ToString().data()});
     return nullptr;
   }
-  return it->second;
+  auto *symbol{it->second};
+  symbol->add_occurrence(component);
+  return symbol;
 }
 
 void ResolveNamesVisitor::Post(const parser::ProcedureDesignator &x) {

flang/lib/semantics/symbol.cc

@@ -156,6 +156,34 @@ const Symbol &Symbol::GetUltimate() const {
   }
 }
 
+const DeclTypeSpec *Symbol::GetType() const {
+  return std::visit(
+      common::visitors{
+          [](const EntityDetails &x) {
+            return x.type().has_value() ? &x.type().value() : nullptr;
+          },
+          [](const ObjectEntityDetails &x) {
+            return x.type().has_value() ? &x.type().value() : nullptr;
+          },
+          [](const ProcEntityDetails &x) { return x.interface().type(); },
+          [](const auto &) {
+            return static_cast<const DeclTypeSpec *>(nullptr);
+          },
+      },
+      details_);
+}
+
+void Symbol::SetType(const DeclTypeSpec &type) {
+  std::visit(
+      common::visitors{
+          [&](EntityDetails &x) { x.set_type(type); },
+          [&](ObjectEntityDetails &x) { x.set_type(type); },
+          [&](ProcEntityDetails &x) { x.interface().set_type(type); },
+          [](auto &) {},
+      },
+      details_);
+}
+
 bool Symbol::isSubprogram() const {
   return std::visit(
       common::visitors{
@@ -224,10 +252,8 @@ std::ostream &operator<<(std::ostream &os, const DerivedTypeDetails &x) {
 }
 
 static std::ostream &DumpType(std::ostream &os, const Symbol &symbol) {
-  if (const auto *details = symbol.detailsIf<EntityDetails>()) {
-    if (details->type()) {
-      os << *details->type() << ' ';
-    }
+  if (const auto *type{symbol.GetType()}) {
+    os << *type << ' ';
   }
   return os;
 }
@@ -314,4 +340,39 @@ std::ostream &operator<<(std::ostream &os, const Symbol &symbol) {
   return os;
 }
 
+// Output a unique name for a scope by qualifying it with the names of
+// parent scopes. For scopes without corresponding symbols, use "ANON".
+static void DumpUniqueName(std::ostream &os, const Scope &scope) {
+  if (&scope != &Scope::globalScope) {
+    DumpUniqueName(os, scope.parent());
+    os << '/';
+    if (auto *scopeSymbol{scope.symbol()}) {
+      os << scopeSymbol->name().ToString();
+    } else {
+      os << "ANON";
+    }
+  }
+}
+
+// Dump a symbol for UnparseWithSymbols. This will be used for tests so the
+// format should be reasonably stable.
+std::ostream &DumpForUnparse(
+    std::ostream &os, const Symbol &symbol, bool isDef) {
+  DumpUniqueName(os, symbol.owner());
+  os << '/' << symbol.name().ToString();
+  if (isDef) {
+    if (!symbol.attrs().empty()) {
+      os << ' ' << symbol.attrs();
+    }
+    if (symbol.test(Symbol::Flag::Implicit)) {
+      os << " (implicit)";
+    }
+    os << ' ' << symbol.GetDetailsName();
+    if (const auto *type{symbol.GetType()}) {
+      os << ' ' << *type;
+    }
+  }
+  return os;
+}
+
 }  // namespace Fortran::semantics

flang/lib/semantics/symbol.h

@@ -277,6 +277,9 @@ public:
   Symbol &GetUltimate();
   const Symbol &GetUltimate() const;
 
+  const DeclTypeSpec *GetType() const;
+  void SetType(const DeclTypeSpec &);
+
   bool isSubprogram() const;
   bool HasExplicitInterface() const;
 
@@ -294,6 +297,7 @@ private:
   Symbol() {}  // only created in class Symbols
   const std::string GetDetailsName() const;
   friend std::ostream &operator<<(std::ostream &, const Symbol &);
+  friend std::ostream &DumpForUnparse(std::ostream &, const Symbol &, bool);
   template<std::size_t> friend class Symbols;
   template<class, std::size_t> friend struct std::array;
 };

flang/lib/semantics/unparse-with-symbols.cc

@@ -0,0 +1,154 @@
+// Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "unparse-with-symbols.h"
+#include "symbol.h"
+#include "../parser/parse-tree-visitor.h"
+#include "../parser/parse-tree.h"
+#include "../parser/unparse.h"
+#include <map>
+#include <ostream>
+#include <set>
+
+namespace Fortran::semantics {
+
+// Walk the parse tree and collection information about which statements
+// define and reference symbols. Then PrintSymbols outputs information
+// by statement.
+class SymbolDumpVisitor {
+public:
+  // Write out symbols defined or referenced at this statement.
+  void PrintSymbols(const parser::CharBlock &stmt, std::ostream &, int) const;
+
+  template<typename T> bool Pre(const T &) { return true; }
+  template<typename T> void Post(const T &) {}
+  template<typename T> bool Pre(const parser::Statement<T> &stmt) {
+    currStmt_ = &stmt.source;
+    return true;
+  }
+  template<typename T> void Post(const parser::Statement<T> &) {
+    currStmt_ = nullptr;
+  }
+  void Post(const parser::Name &);
+  void Post(const parser::EndFunctionStmt &) { EndScope(); }
+  void Post(const parser::EndModuleStmt &) { EndScope(); }
+  void Post(const parser::EndMpSubprogramStmt &) { EndScope(); }
+  void Post(const parser::EndProgramStmt &) { EndScope(); }
+  void Post(const parser::EndSubmoduleStmt &) { EndScope(); }
+  void Post(const parser::EndSubroutineStmt &) { EndScope(); }
+  bool Pre(const parser::DataRef &) { return PreReference(); }
+  void Post(const parser::DataRef &) { PostReference(); }
+  bool Pre(const parser::Designator &) { return PreReference(); }
+  void Post(const parser::Designator &) { PostReference(); }
+  bool Pre(const parser::StructureComponent &) { return PreReference(); }
+  void Post(const parser::StructureComponent &) { PostReference(); }
+  bool Pre(const parser::ProcedureDesignator &) { return PreReference(); }
+  void Post(const parser::ProcedureDesignator &) { PostReference(); }
+  bool Pre(const parser::DerivedTypeSpec &) { return PreReference(); }
+  void Post(const parser::DerivedTypeSpec &) { PostReference(); }
+  bool Pre(const parser::UseStmt &) { return PreReference(); }
+  void Post(const parser::UseStmt &) { PostReference(); }
+
+private:
+  using symbolMap = std::multimap<const char *, const Symbol *>;
+
+  const SourceName *currStmt_{nullptr};  // current statement we are processing
+  int isRef_{0};  // > 0 means in the context of a reference
+  symbolMap defs_;  // statement location to symbol defined there
+  symbolMap refs_;  // statement location to symbol referenced there
+  std::map<const Symbol *, const char *> symbolToStmt_;  // symbol to def
+
+  void EndScope();
+  bool PreReference();
+  void PostReference();
+  bool isRef() const { return isRef_ > 0; }
+  void PrintSymbols(const parser::CharBlock &, std::ostream &, int,
+      const symbolMap &, bool) const;
+  void Indent(std::ostream &, int) const;
+};
+
+void SymbolDumpVisitor::PrintSymbols(
+    const parser::CharBlock &location, std::ostream &out, int indent) const {
+  PrintSymbols(location, out, indent, defs_, true);
+  PrintSymbols(location, out, indent, refs_, false);
+}
+void SymbolDumpVisitor::PrintSymbols(const parser::CharBlock &location,
+    std::ostream &out, int indent, const symbolMap &symbols, bool isDef) const {
+  std::set<const Symbol *> done;  // used to prevent duplicates
+  auto range{symbols.equal_range(location.begin())};
+  for (auto it{range.first}; it != range.second; ++it) {
+    auto *symbol{it->second};
+    if (done.insert(symbol).second) {
+      Indent(out, indent);
+      out << '!' << (isDef ? "DEF"s : "REF"s) << ": ";
+      DumpForUnparse(out, symbol->GetUltimate(), isDef);
+      out << '\n';
+    }
+  }
+}
+void SymbolDumpVisitor::Indent(std::ostream &out, int indent) const {
+  for (int i{0}; i < indent; ++i) {
+    out << ' ';
+  }
+}
+
+void SymbolDumpVisitor::Post(const parser::Name &name) {
+  if (const auto *symbol{name.symbol}) {
+    CHECK(currStmt_);
+    // If this is the first reference to an implicitly defined symbol,
+    // record it as a def.
+    bool isImplicit{symbol->test(Symbol::Flag::Implicit) &&
+        symbolToStmt_.find(symbol) == symbolToStmt_.end()};
+    if (isRef() && !isImplicit) {
+      refs_.emplace(currStmt_->begin(), symbol);
+    } else {
+      symbolToStmt_.emplace(symbol, currStmt_->begin());
+    }
+  }
+}
+
+// Defs are initially saved in symbolToStmt_ so that a symbol defined across
+// multiple statements is associated with only one (the first). Now that we
+// are at the end of a scope, move them into defs_.
+void SymbolDumpVisitor::EndScope() {
+  for (auto pair : symbolToStmt_) {
+    defs_.emplace(pair.second, pair.first);
+  }
+  symbolToStmt_.clear();
+}
+
+// {Pre,Post}Reference() are called around constructs that contains symbols
+// references. Sometimes those are nested (e.g. DataRef inside Designator)
+// so we need to maintain a count to know when we are back out.
+bool SymbolDumpVisitor::PreReference() {
+  ++isRef_;
+  return true;
+}
+void SymbolDumpVisitor::PostReference() {
+  CHECK(isRef_ > 0);
+  --isRef_;
+}
+
+void UnparseWithSymbols(std::ostream &out, const parser::Program &program,
+    parser::Encoding encoding) {
+  SymbolDumpVisitor visitor;
+  parser::Walk(program, visitor);
+  parser::preStatementType preStatement{
+      [&](const parser::CharBlock &location, std::ostream &out, int indent) {
+        visitor.PrintSymbols(location, out, indent);
+      }};
+  parser::Unparse(out, program, encoding, false, &preStatement);
+}
+
+}  // namespace Fortran::semantics

flang/lib/semantics/unparse-with-symbols.h

@@ -0,0 +1,30 @@
+// Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef FORTRAN_SEMANTICS_UNPARSE_WITH_SYMBOLS_H_
+#define FORTRAN_SEMANTICS_UNPARSE_WITH_SYMBOLS_H_
+
+#include "../parser/characters.h"
+#include <iosfwd>
+
+namespace Fortran::parser {
+struct Program;
+}  // namespace Fortran::parser
+
+namespace Fortran::semantics {
+void UnparseWithSymbols(std::ostream &, const parser::Program &,
+    parser::Encoding encoding = parser::Encoding::UTF8);
+}  // namespace Fortran::semantics
+
+#endif  // FORTRAN_SEMANTICS_UNPARSE_WITH_SYMBOLS_H_

flang/tools/f18/f18.cc

@@ -23,6 +23,7 @@
 #include "../../lib/parser/unparse.h"
 #include "../../lib/semantics/dump-parse-tree.h"
 #include "../../lib/semantics/resolve-names.h"
+#include "../../lib/semantics/unparse-with-symbols.h"
 #include <cerrno>
 #include <cstdio>
 #include <cstring>
@@ -196,23 +197,29 @@ std::string CompileFortran(
     exitStatus = EXIT_FAILURE;
     return {};
   }
+  auto &parseTree{*parsing.parseTree()};
   if (driver.measureTree) {
-    MeasureParseTree(*parsing.parseTree());
+    MeasureParseTree(parseTree);
+  }
+  if (driver.dumpParseTree) {
+    Fortran::semantics::DumpTree(std::cout, parseTree);
+  }
+  if (driver.dumpUnparse) {
+    if (driver.dumpSymbols) {
+      Fortran::semantics::ResolveNames(parseTree, parsing.cooked());
+      Fortran::semantics::UnparseWithSymbols(
+          std::cout, parseTree, driver.encoding);
+    } else {
+      Unparse(std::cout, parseTree, driver.encoding, true);
+    }
+    return {};
   }
   if (driver.debugResolveNames || driver.dumpSymbols) {
-    Fortran::semantics::ResolveNames(*parsing.parseTree(), parsing.cooked());
+    Fortran::semantics::ResolveNames(parseTree, parsing.cooked());
     if (driver.dumpSymbols) {
       Fortran::semantics::DumpSymbols(std::cout);
     }
   }
-  if (driver.dumpParseTree) {
-    Fortran::semantics::DumpTree(std::cout, *parsing.parseTree());
-  }
-  if (driver.dumpUnparse) {
-    Unparse(
-        std::cout, *parsing.parseTree(), driver.encoding, true /*capitalize*/);
-    return {};
-  }
   if (driver.parseOnly) {
     return {};
   }
@@ -225,7 +232,7 @@ std::string CompileFortran(
   {
     std::ofstream tmpSource;
     tmpSource.open(tmpSourcePath);
-    Unparse(tmpSource, *parsing.parseTree(), driver.encoding);
+    Unparse(tmpSource, parseTree, driver.encoding);
   }
 
   if (ParentProcess()) {