[ORC] Re-apply r321838 again with a workaround for a bug present in the libcxx

version being used on some of the green dragon builders (plus a clang-format).

Workaround: AsynchronousSymbolQuery and VSO want to work with
JITEvaluatedSymbols anyway, so just use them (instead of JITSymbol, which
happens to tickle the bug).

The libcxx bug being worked around was fixed in r276003, and there are plans to
update the offending builders.

llvm-svn: 322140

llvm/include/llvm/ExecutionEngine/JITSymbol.h

@@ -145,6 +145,8 @@ private:
 /// @brief Represents a symbol that has been evaluated to an address already.
 class JITEvaluatedSymbol {
 public:
+  JITEvaluatedSymbol() = default;
+
   /// @brief Create a 'null' symbol.
   JITEvaluatedSymbol(std::nullptr_t) {}
 

llvm/include/llvm/ExecutionEngine/Orc/Core.h

@@ -0,0 +1,235 @@
+//===------ Core.h -- Core ORC APIs (Layer, JITDylib, etc.) -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Contains core ORC APIs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_CORE_H
+#define LLVM_EXECUTIONENGINE_ORC_CORE_H
+
+#include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
+
+#include <deque>
+#include <map>
+#include <memory>
+#include <set>
+#include <vector>
+
+namespace llvm {
+namespace orc {
+
+class VSO;
+
+/// @brief A set of symbol names (represented by SymbolStringPtrs for
+//         efficiency).
+using SymbolNameSet = std::set<SymbolStringPtr>;
+
+/// @brief A map from symbol names (as SymbolStringPtrs) to JITSymbols
+///        (address/flags pairs).
+using SymbolMap = std::map<SymbolStringPtr, JITEvaluatedSymbol>;
+
+/// @brief A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.
+using SymbolFlagsMap = std::map<SymbolStringPtr, JITSymbolFlags>;
+
+/// @brief A symbol query that returns results via a callback when results are
+///        ready.
+///
+/// makes a callback when all symbols are available.
+class AsynchronousSymbolQuery {
+public:
+  /// @brief Callback to notify client that symbols have been resolved.
+  using SymbolsResolvedCallback = std::function<void(Expected<SymbolMap>)>;
+
+  /// @brief Callback to notify client that symbols are ready for execution.
+  using SymbolsReadyCallback = std::function<void(Error)>;
+
+  /// @brief Create a query for the given symbols, notify-resolved and
+  ///        notify-ready callbacks.
+  AsynchronousSymbolQuery(const SymbolNameSet &Symbols,
+                          SymbolsResolvedCallback NotifySymbolsResolved,
+                          SymbolsReadyCallback NotifySymbolsReady);
+
+  /// @brief Notify client that the query failed.
+  ///
+  /// If the notify-resolved callback has not been made yet, then it is called
+  /// with the given error, and the notify-finalized callback is never made.
+  ///
+  /// If the notify-resolved callback has already been made then then the
+  /// notify-finalized callback is called with the given error.
+  ///
+  /// It is illegal to call setFailed after both callbacks have been made.
+  void setFailed(Error Err);
+
+  /// @brief Set the resolved symbol information for the given symbol name.
+  ///
+  /// If this symbol was the last one not resolved, this will trigger a call to
+  /// the notify-finalized callback passing the completed sybol map.
+  void setDefinition(SymbolStringPtr Name, JITEvaluatedSymbol Sym);
+
+  /// @brief Notify the query that a requested symbol is ready for execution.
+  ///
+  /// This decrements the query's internal count of not-yet-ready symbols. If
+  /// this call to notifySymbolFinalized sets the counter to zero, it will call
+  /// the notify-finalized callback with Error::success as the value.
+  void notifySymbolFinalized();
+
+private:
+  SymbolMap Symbols;
+  size_t OutstandingResolutions = 0;
+  size_t OutstandingFinalizations = 0;
+  SymbolsResolvedCallback NotifySymbolsResolved;
+  SymbolsReadyCallback NotifySymbolsReady;
+};
+
+/// @brief Represents a source of symbol definitions which may be materialized
+///        (turned into data / code through some materialization process) or
+///        discarded (if the definition is overridden by a stronger one).
+///
+/// SymbolSources are used when providing lazy definitions of symbols to VSOs.
+/// The VSO will call materialize when the address of a symbol is requested via
+/// the lookup method. The VSO will call discard if a stronger definition is
+/// added or already present.
+class SymbolSource {
+public:
+  virtual ~SymbolSource() {}
+
+  /// @brief Implementations of this method should materialize the given
+  ///        symbols (plus any additional symbols required) by adding a
+  ///        Materializer to the ExecutionSession's MaterializationQueue.
+  virtual Error materialize(VSO &V, SymbolNameSet Symbols) = 0;
+
+  /// @brief Implementations of this method should discard the given symbol
+  ///        from the source (e.g. if the source is an LLVM IR Module and the
+  ///        symbol is a function, delete the function body or mark it available
+  ///        externally).
+  virtual void discard(VSO &V, SymbolStringPtr Name) = 0;
+
+private:
+  virtual void anchor();
+};
+
+/// @brief Represents a dynamic linkage unit in a JIT process.
+///
+/// VSO acts as a symbol table (symbol definitions can be set and the dylib
+/// queried to find symbol addresses) and as a key for tracking resources
+/// (since a VSO's address is fixed).
+class VSO {
+  friend class ExecutionSession;
+
+public:
+  /// @brief
+  enum RelativeLinkageStrength {
+    NewDefinitionIsStronger,
+    DuplicateDefinition,
+    ExistingDefinitionIsStronger
+  };
+
+  using SetDefinitionsResult =
+      std::map<SymbolStringPtr, RelativeLinkageStrength>;
+  using SourceWorkMap = std::map<SymbolSource *, SymbolNameSet>;
+
+  struct LookupResult {
+    SourceWorkMap MaterializationWork;
+    SymbolNameSet UnresolvedSymbols;
+  };
+
+  VSO() = default;
+
+  VSO(const VSO &) = delete;
+  VSO &operator=(const VSO &) = delete;
+  VSO(VSO &&) = delete;
+  VSO &operator=(VSO &&) = delete;
+
+  /// @brief Compare new linkage with existing linkage.
+  static RelativeLinkageStrength
+  compareLinkage(Optional<JITSymbolFlags> OldFlags, JITSymbolFlags NewFlags);
+
+  /// @brief Compare new linkage with an existing symbol's linkage.
+  RelativeLinkageStrength compareLinkage(SymbolStringPtr Name,
+                                         JITSymbolFlags NewFlags) const;
+
+  /// @brief Adds the given symbols to the mapping as resolved, finalized
+  ///        symbols.
+  ///
+  /// FIXME: We can take this by const-ref once symbol-based laziness is
+  ///        removed.
+  Error define(SymbolMap NewSymbols);
+
+  /// @brief Adds the given symbols to the mapping as lazy symbols.
+  Error defineLazy(const SymbolFlagsMap &NewSymbols, SymbolSource &Source);
+
+  /// @brief Add the given symbol/address mappings to the dylib, but do not
+  ///        mark the symbols as finalized yet.
+  void resolve(SymbolMap SymbolValues);
+
+  /// @brief Finalize the given symbols.
+  void finalize(SymbolNameSet SymbolsToFinalize);
+
+  /// @brief Apply the given query to the given symbols in this VSO.
+  ///
+  /// For symbols in this VSO that have already been materialized, their address
+  /// will be set in the query immediately.
+  ///
+  /// For symbols in this VSO that have not been materialized, the query will be
+  /// recorded and the source for those symbols (plus the set of symbols to be
+  /// materialized by that source) will be returned as the MaterializationWork
+  /// field of the LookupResult.
+  ///
+  /// Any symbols not found in this VSO will be returned in the
+  /// UnresolvedSymbols field of the LookupResult.
+  LookupResult lookup(AsynchronousSymbolQuery &Query, SymbolNameSet Symbols);
+
+private:
+  class MaterializationInfo {
+  public:
+    MaterializationInfo(JITSymbolFlags Flags, AsynchronousSymbolQuery &Query);
+    JITSymbolFlags getFlags() const;
+    JITTargetAddress getAddress() const;
+    void query(SymbolStringPtr Name, AsynchronousSymbolQuery &Query);
+    void resolve(SymbolStringPtr Name, JITEvaluatedSymbol Sym);
+    void finalize();
+
+  private:
+    JITSymbolFlags Flags;
+    JITTargetAddress Address = 0;
+    std::vector<AsynchronousSymbolQuery *> PendingResolution;
+    std::vector<AsynchronousSymbolQuery *> PendingFinalization;
+  };
+
+  class SymbolTableEntry {
+  public:
+    SymbolTableEntry(JITSymbolFlags Flags, SymbolSource &Source);
+    SymbolTableEntry(JITEvaluatedSymbol Sym);
+    SymbolTableEntry(SymbolTableEntry &&Other);
+    ~SymbolTableEntry();
+    JITSymbolFlags getFlags() const;
+    void replaceWithSource(VSO &V, SymbolStringPtr Name, JITSymbolFlags Flags,
+                           SymbolSource &NewSource);
+    SymbolSource *query(SymbolStringPtr Name, AsynchronousSymbolQuery &Query);
+    void resolve(VSO &V, SymbolStringPtr Name, JITEvaluatedSymbol Sym);
+    void finalize();
+
+  private:
+    JITSymbolFlags Flags;
+    union {
+      JITTargetAddress Address;
+      SymbolSource *Source;
+      std::unique_ptr<MaterializationInfo> MatInfo;
+    };
+  };
+
+  std::map<SymbolStringPtr, SymbolTableEntry> Symbols;
+};
+
+} // End namespace orc
+} // End namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_CORE_H

llvm/lib/ExecutionEngine/Orc/CMakeLists.txt

@@ -1,4 +1,5 @@
 add_llvm_library(LLVMOrcJIT
+  Core.cpp
   ExecutionUtils.cpp
   IndirectionUtils.cpp
   NullResolver.cpp

llvm/lib/ExecutionEngine/Orc/Core.cpp

@@ -0,0 +1,317 @@
+//===--------- Core.cpp - Core ORC APIs (SymbolSource, VSO, etc.) ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/OrcError.h"
+
+namespace llvm {
+namespace orc {
+
+void SymbolSource::anchor() {}
+
+AsynchronousSymbolQuery::AsynchronousSymbolQuery(
+    const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved,
+    SymbolsReadyCallback NotifySymbolsReady)
+    : NotifySymbolsResolved(std::move(NotifySymbolsResolved)),
+      NotifySymbolsReady(std::move(NotifySymbolsReady)) {
+  assert(this->NotifySymbolsResolved &&
+         "Symbols resolved callback must be set");
+  assert(this->NotifySymbolsReady && "Symbols ready callback must be set");
+  OutstandingResolutions = OutstandingFinalizations = Symbols.size();
+}
+
+void AsynchronousSymbolQuery::setFailed(Error Err) {
+  OutstandingResolutions = OutstandingFinalizations = 0;
+  if (NotifySymbolsResolved)
+    NotifySymbolsResolved(std::move(Err));
+  else
+    NotifySymbolsReady(std::move(Err));
+}
+
+void AsynchronousSymbolQuery::setDefinition(SymbolStringPtr Name,
+                                            JITEvaluatedSymbol Sym) {
+  // If OutstandingResolutions is zero we must have errored out already. Just
+  // ignore this.
+  if (OutstandingResolutions == 0)
+    return;
+
+  assert(!Symbols.count(Name) && "Symbol has already been assigned an address");
+  Symbols.insert(std::make_pair(std::move(Name), std::move(Sym)));
+  --OutstandingResolutions;
+  if (OutstandingResolutions == 0) {
+    NotifySymbolsResolved(std::move(Symbols));
+    // Null out NotifySymbolsResolved to indicate that we've already called it.
+    NotifySymbolsResolved = {};
+  }
+}
+
+void AsynchronousSymbolQuery::notifySymbolFinalized() {
+  // If OutstandingFinalizations is zero we must have errored out already. Just
+  // ignore this.
+  if (OutstandingFinalizations == 0)
+    return;
+
+  assert(OutstandingFinalizations > 0 && "All symbols already finalized");
+  --OutstandingFinalizations;
+  if (OutstandingFinalizations == 0)
+    NotifySymbolsReady(Error::success());
+}
+
+VSO::MaterializationInfo::MaterializationInfo(JITSymbolFlags Flags,
+                                              AsynchronousSymbolQuery &Query)
+    : Flags(std::move(Flags)), PendingResolution({&Query}) {}
+
+JITSymbolFlags VSO::MaterializationInfo::getFlags() const { return Flags; }
+
+JITTargetAddress VSO::MaterializationInfo::getAddress() const {
+  return Address;
+}
+
+void VSO::MaterializationInfo::query(SymbolStringPtr Name,
+                                     AsynchronousSymbolQuery &Query) {
+  if (Address != 0) {
+    Query.setDefinition(Name, JITEvaluatedSymbol(Address, Flags));
+    PendingFinalization.push_back(&Query);
+  } else
+    PendingResolution.push_back(&Query);
+}
+
+void VSO::MaterializationInfo::resolve(SymbolStringPtr Name,
+                                       JITEvaluatedSymbol Sym) {
+  // FIXME: Sanity check flags?
+  Flags = Sym.getFlags();
+  Address = Sym.getAddress();
+  for (auto *Query : PendingResolution) {
+    Query->setDefinition(Name, std::move(Sym));
+    PendingFinalization.push_back(Query);
+  }
+  PendingResolution = {};
+}
+
+void VSO::MaterializationInfo::finalize() {
+  for (auto *Query : PendingFinalization)
+    Query->notifySymbolFinalized();
+  PendingFinalization = {};
+}
+
+VSO::SymbolTableEntry::SymbolTableEntry(JITSymbolFlags Flags,
+                                        SymbolSource &Source)
+    : Flags(JITSymbolFlags::FlagNames(Flags | JITSymbolFlags::NotMaterialized)),
+      Source(&Source) {
+  // FIXME: Assert flag sanity.
+}
+
+VSO::SymbolTableEntry::SymbolTableEntry(JITEvaluatedSymbol Sym)
+    : Flags(Sym.getFlags()), Address(Sym.getAddress()) {
+  // FIXME: Assert flag sanity.
+}
+
+VSO::SymbolTableEntry::SymbolTableEntry(SymbolTableEntry &&Other)
+    : Flags(Other.Flags), Address(0) {
+  if (Flags.isMaterializing())
+    MatInfo = std::move(Other.MatInfo);
+  else
+    Source = Other.Source;
+}
+
+VSO::SymbolTableEntry::~SymbolTableEntry() {
+  assert(!Flags.isMaterializing() &&
+         "Symbol table entry destroyed while symbol was being materialized");
+}
+
+JITSymbolFlags VSO::SymbolTableEntry::getFlags() const { return Flags; }
+
+void VSO::SymbolTableEntry::replaceWithSource(VSO &V, SymbolStringPtr Name,
+                                              JITSymbolFlags Flags,
+                                              SymbolSource &NewSource) {
+  assert(!this->Flags.isMaterializing() &&
+         "Attempted to replace symbol with lazy definition during "
+         "materialization");
+  if (!this->Flags.isMaterialized())
+    Source->discard(V, Name);
+  this->Flags = Flags;
+  this->Source = &NewSource;
+}
+
+SymbolSource *VSO::SymbolTableEntry::query(SymbolStringPtr Name,
+                                           AsynchronousSymbolQuery &Query) {
+  if (Flags.isMaterializing()) {
+    MatInfo->query(std::move(Name), Query);
+    return nullptr;
+  } else if (Flags.isMaterialized()) {
+    Query.setDefinition(std::move(Name), JITEvaluatedSymbol(Address, Flags));
+    Query.notifySymbolFinalized();
+    return nullptr;
+  }
+  SymbolSource *S = Source;
+  new (&MatInfo) std::unique_ptr<MaterializationInfo>(
+      llvm::make_unique<MaterializationInfo>(Flags, Query));
+  Flags |= JITSymbolFlags::Materializing;
+  return S;
+}
+
+void VSO::SymbolTableEntry::resolve(VSO &V, SymbolStringPtr Name,
+                                    JITEvaluatedSymbol Sym) {
+  if (Flags.isMaterializing())
+    MatInfo->resolve(std::move(Name), std::move(Sym));
+  else {
+    // If there's a layer for this symbol.
+    if (!Flags.isMaterialized())
+      Source->discard(V, Name);
+
+    // FIXME: Should we assert flag state here (flags must match except for
+    //        materialization state, overrides must be legal) or in the caller
+    //        in VSO?
+    Flags = Sym.getFlags();
+    Address = Sym.getAddress();
+  }
+}
+
+void VSO::SymbolTableEntry::finalize() {
+  if (Flags.isMaterializing()) {
+    auto TmpMatInfo = std::move(MatInfo);
+    MatInfo.std::unique_ptr<MaterializationInfo>::~unique_ptr();
+    // FIXME: Assert flag sanity?
+    Flags = TmpMatInfo->getFlags();
+    Address = TmpMatInfo->getAddress();
+    TmpMatInfo->finalize();
+  }
+  assert(Flags.isMaterialized() && "Trying to finalize not-emitted symbol");
+}
+
+VSO::RelativeLinkageStrength VSO::compareLinkage(Optional<JITSymbolFlags> Old,
+                                                 JITSymbolFlags New) {
+  if (Old == None)
+    return llvm::orc::VSO::NewDefinitionIsStronger;
+
+  if (Old->isStrongDefinition()) {
+    if (New.isStrongDefinition())
+      return llvm::orc::VSO::DuplicateDefinition;
+    else
+      return llvm::orc::VSO::ExistingDefinitionIsStronger;
+  } else {
+    if (New.isStrongDefinition())
+      return llvm::orc::VSO::NewDefinitionIsStronger;
+    else
+      return llvm::orc::VSO::ExistingDefinitionIsStronger;
+  }
+}
+
+VSO::RelativeLinkageStrength
+VSO::compareLinkage(SymbolStringPtr Name, JITSymbolFlags NewFlags) const {
+  auto I = Symbols.find(Name);
+  return compareLinkage(I == Symbols.end()
+                            ? None
+                            : Optional<JITSymbolFlags>(I->second.getFlags()),
+                        NewFlags);
+}
+
+Error VSO::define(SymbolMap NewSymbols) {
+  Error Err = Error::success();
+  for (auto &KV : NewSymbols) {
+    auto I = Symbols.find(KV.first);
+    auto LinkageResult = compareLinkage(
+        I == Symbols.end() ? None
+                           : Optional<JITSymbolFlags>(I->second.getFlags()),
+        KV.second.getFlags());
+
+    // Silently discard weaker definitions.
+    if (LinkageResult == ExistingDefinitionIsStronger)
+      continue;
+
+    // Report duplicate definition errors.
+    if (LinkageResult == DuplicateDefinition) {
+      Err = joinErrors(std::move(Err),
+                       make_error<orc::DuplicateDefinition>(*KV.first));
+      continue;
+    }
+
+    if (I != Symbols.end()) {
+      I->second.resolve(*this, KV.first, std::move(KV.second));
+      I->second.finalize();
+    } else
+      Symbols.insert(std::make_pair(KV.first, std::move(KV.second)));
+  }
+  return Err;
+}
+
+Error VSO::defineLazy(const SymbolFlagsMap &NewSymbols, SymbolSource &Source) {
+  Error Err = Error::success();
+  for (auto &KV : NewSymbols) {
+    auto I = Symbols.find(KV.first);
+
+    auto LinkageResult = compareLinkage(
+        I == Symbols.end() ? None
+                           : Optional<JITSymbolFlags>(I->second.getFlags()),
+        KV.second);
+
+    // Discard weaker definitions.
+    if (LinkageResult == ExistingDefinitionIsStronger)
+      Source.discard(*this, KV.first);
+
+    // Report duplicate definition errors.
+    if (LinkageResult == DuplicateDefinition) {
+      Err = joinErrors(std::move(Err),
+                       make_error<orc::DuplicateDefinition>(*KV.first));
+      continue;
+    }
+
+    if (I != Symbols.end())
+      I->second.replaceWithSource(*this, KV.first, KV.second, Source);
+    else
+      Symbols.emplace(
+          std::make_pair(KV.first, SymbolTableEntry(KV.second, Source)));
+  }
+  return Err;
+}
+
+void VSO::resolve(SymbolMap SymbolValues) {
+  for (auto &KV : SymbolValues) {
+    auto I = Symbols.find(KV.first);
+    assert(I != Symbols.end() && "Resolving symbol not present in this dylib");
+    I->second.resolve(*this, KV.first, std::move(KV.second));
+  }
+}
+
+void VSO::finalize(SymbolNameSet SymbolsToFinalize) {
+  for (auto &S : SymbolsToFinalize) {
+    auto I = Symbols.find(S);
+    assert(I != Symbols.end() && "Finalizing symbol not present in this dylib");
+    I->second.finalize();
+  }
+}
+
+VSO::LookupResult VSO::lookup(AsynchronousSymbolQuery &Query,
+                              SymbolNameSet Names) {
+  SourceWorkMap MaterializationWork;
+
+  for (SymbolNameSet::iterator I = Names.begin(), E = Names.end(); I != E;) {
+    auto Tmp = I;
+    ++I;
+    auto SymI = Symbols.find(*Tmp);
+
+    // If the symbol isn't in this dylib then just continue.
+    // If it is, erase it from Names and proceed.
+    if (SymI == Symbols.end())
+      continue;
+    else
+      Names.erase(Tmp);
+
+    // Forward the query to the given SymbolTableEntry, and if it return a
+    // layer to perform materialization with, add that to the
+    // MaterializationWork map.
+    if (auto *Source = SymI->second.query(SymI->first, Query))
+      MaterializationWork[Source].insert(SymI->first);
+  }
+
+  return {std::move(MaterializationWork), std::move(Names)};
+}
+
+} // End namespace orc.
+} // End namespace llvm.

llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt

@@ -11,6 +11,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(OrcJITTests
   CompileOnDemandLayerTest.cpp
+  CoreAPIsTest.cpp
   IndirectionUtilsTest.cpp
   GlobalMappingLayerTest.cpp
   LazyEmittingLayerTest.cpp

llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp

@@ -0,0 +1,211 @@
+//===----------- CoreAPIsTest.cpp - Unit tests for Core ORC APIs ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "OrcTestCommon.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "gtest/gtest.h"
+
+#include <set>
+
+using namespace llvm;
+using namespace llvm::orc;
+
+namespace {
+
+class SimpleSource : public SymbolSource {
+public:
+  using MaterializeFunction = std::function<Error(VSO &, SymbolNameSet)>;
+  using DiscardFunction = std::function<void(VSO &, SymbolStringPtr)>;
+
+  SimpleSource(MaterializeFunction Materialize, DiscardFunction Discard)
+      : Materialize(std::move(Materialize)), Discard(std::move(Discard)) {}
+
+  Error materialize(VSO &V, SymbolNameSet Symbols) override {
+    return Materialize(V, std::move(Symbols));
+  }
+
+  void discard(VSO &V, SymbolStringPtr Name) override {
+    Discard(V, std::move(Name));
+  }
+
+private:
+  MaterializeFunction Materialize;
+  DiscardFunction Discard;
+};
+
+TEST(CoreAPIsTest, AsynchronousSymbolQuerySuccessfulResolutionOnly) {
+  SymbolStringPool SP;
+  auto Foo = SP.intern("foo");
+  constexpr JITTargetAddress FakeAddr = 0xdeadbeef;
+  SymbolNameSet Names({Foo});
+
+  bool OnResolutionRun = false;
+  bool OnReadyRun = false;
+  auto OnResolution = [&](Expected<SymbolMap> Result) {
+    EXPECT_TRUE(!!Result) << "Resolution unexpectedly returned error";
+    auto I = Result->find(Foo);
+    EXPECT_NE(I, Result->end()) << "Could not find symbol definition";
+    EXPECT_EQ(I->second.getAddress(), FakeAddr)
+        << "Resolution returned incorrect result";
+    OnResolutionRun = true;
+  };
+  auto OnReady = [&](Error Err) {
+    cantFail(std::move(Err));
+    OnReadyRun = true;
+  };
+
+  AsynchronousSymbolQuery Q(Names, OnResolution, OnReady);
+
+  Q.setDefinition(Foo, JITEvaluatedSymbol(FakeAddr, JITSymbolFlags::Exported));
+
+  EXPECT_TRUE(OnResolutionRun) << "OnResolutionCallback was not run";
+  EXPECT_FALSE(OnReadyRun) << "OnReady unexpectedly run";
+}
+
+TEST(CoreAPIsTest, AsynchronousSymbolQueryResolutionErrorOnly) {
+  SymbolStringPool SP;
+  auto Foo = SP.intern("foo");
+  SymbolNameSet Names({Foo});
+
+  bool OnResolutionRun = false;
+  bool OnReadyRun = false;
+
+  auto OnResolution = [&](Expected<SymbolMap> Result) {
+    EXPECT_FALSE(!!Result) << "Resolution unexpectedly returned success";
+    auto Msg = toString(Result.takeError());
+    EXPECT_EQ(Msg, "xyz") << "Resolution returned incorrect result";
+    OnResolutionRun = true;
+  };
+  auto OnReady = [&](Error Err) {
+    cantFail(std::move(Err));
+    OnReadyRun = true;
+  };
+
+  AsynchronousSymbolQuery Q(Names, OnResolution, OnReady);
+
+  Q.setFailed(make_error<StringError>("xyz", inconvertibleErrorCode()));
+
+  EXPECT_TRUE(OnResolutionRun) << "OnResolutionCallback was not run";
+  EXPECT_FALSE(OnReadyRun) << "OnReady unexpectedly run";
+}
+
+TEST(CoreAPIsTest, SimpleAsynchronousSymbolQueryAgainstVSO) {
+  SymbolStringPool SP;
+  auto Foo = SP.intern("foo");
+  constexpr JITTargetAddress FakeAddr = 0xdeadbeef;
+  SymbolNameSet Names({Foo});
+
+  bool OnResolutionRun = false;
+  bool OnReadyRun = false;
+
+  auto OnResolution = [&](Expected<SymbolMap> Result) {
+    EXPECT_TRUE(!!Result) << "Query unexpectedly returned error";
+    auto I = Result->find(Foo);
+    EXPECT_NE(I, Result->end()) << "Could not find symbol definition";
+    EXPECT_EQ(I->second.getAddress(), FakeAddr)
+        << "Resolution returned incorrect result";
+    OnResolutionRun = true;
+  };
+
+  auto OnReady = [&](Error Err) {
+    cantFail(std::move(Err));
+    OnReadyRun = true;
+  };
+
+  AsynchronousSymbolQuery Q(Names, OnResolution, OnReady);
+  VSO V;
+
+  SymbolMap Defs;
+  Defs[Foo] = JITEvaluatedSymbol(FakeAddr, JITSymbolFlags::Exported);
+  cantFail(V.define(std::move(Defs)));
+  V.lookup(Q, Names);
+
+  EXPECT_TRUE(OnResolutionRun) << "OnResolutionCallback was not run";
+  EXPECT_TRUE(OnReadyRun) << "OnReady was not run";
+}
+
+TEST(CoreAPIsTest, AddAndMaterializeLazySymbol) {
+
+  constexpr JITTargetAddress FakeFooAddr = 0xdeadbeef;
+  constexpr JITTargetAddress FakeBarAddr = 0xcafef00d;
+
+  SymbolStringPool SP;
+  auto Foo = SP.intern("foo");
+  auto Bar = SP.intern("bar");
+
+  bool FooMaterialized = false;
+  bool BarDiscarded = false;
+
+  VSO V;
+
+  auto Source = std::make_shared<SimpleSource>(
+      [&](VSO &V, SymbolNameSet Symbols) {
+        EXPECT_EQ(Symbols.size(), 1U)
+            << "Expected Symbols set size to be 1 ({ Foo })";
+        EXPECT_EQ(*Symbols.begin(), Foo) << "Expected Symbols == { Foo }";
+
+        SymbolMap SymbolsToResolve;
+        SymbolsToResolve[Foo] =
+            JITEvaluatedSymbol(FakeFooAddr, JITSymbolFlags::Exported);
+        V.resolve(std::move(SymbolsToResolve));
+        SymbolNameSet SymbolsToFinalize;
+        SymbolsToFinalize.insert(Foo);
+        V.finalize(SymbolsToFinalize);
+        FooMaterialized = true;
+        return Error::success();
+      },
+      [&](VSO &V, SymbolStringPtr Name) {
+        EXPECT_EQ(Name, Bar) << "Expected Name to be Bar";
+        BarDiscarded = true;
+      });
+
+  SymbolFlagsMap InitialSymbols(
+      {{Foo, JITSymbolFlags::Exported},
+       {Bar, static_cast<JITSymbolFlags::FlagNames>(JITSymbolFlags::Exported |
+                                                    JITSymbolFlags::Weak)}});
+  cantFail(V.defineLazy(InitialSymbols, *Source));
+
+  SymbolMap BarOverride;
+  BarOverride[Bar] = JITEvaluatedSymbol(FakeBarAddr, JITSymbolFlags::Exported);
+  cantFail(V.define(std::move(BarOverride)));
+
+  SymbolNameSet Names({Foo});
+
+  bool OnResolutionRun = false;
+  bool OnReadyRun = false;
+
+  auto OnResolution = [&](Expected<SymbolMap> Result) {
+    EXPECT_TRUE(!!Result) << "Resolution unexpectedly returned error";
+    auto I = Result->find(Foo);
+    EXPECT_NE(I, Result->end()) << "Could not find symbol definition";
+    EXPECT_EQ(I->second.getAddress(), FakeFooAddr)
+        << "Resolution returned incorrect result";
+    OnResolutionRun = true;
+  };
+
+  auto OnReady = [&](Error Err) {
+    cantFail(std::move(Err));
+    OnReadyRun = true;
+  };
+
+  AsynchronousSymbolQuery Q(Names, OnResolution, OnReady);
+
+  auto LR = V.lookup(Q, Names);
+
+  for (auto &SWKV : LR.MaterializationWork)
+    cantFail(SWKV.first->materialize(V, std::move(SWKV.second)));
+
+  EXPECT_TRUE(LR.UnresolvedSymbols.empty()) << "Could not find Foo in dylib";
+  EXPECT_TRUE(FooMaterialized) << "Foo was not materialized";
+  EXPECT_TRUE(BarDiscarded) << "Bar was not discarded";
+  EXPECT_TRUE(OnResolutionRun) << "OnResolutionCallback was not run";
+  EXPECT_TRUE(OnReadyRun) << "OnReady was not run";
+}
+
+} // namespace