[ORC] Enable JITEventListeners in the RTDyldObjectLinkingLayer.

Enable use of ExecutionEngine JITEventListeners in RTDyldObjectLinkingLayer.
This allows existing MCJIT clients to more easily migrate to LLJIT / ORCv2.

Example usage in llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener.

Differential Revision: https://reviews.llvm.org/D75838

6 files changed, 191 insertions, 11 deletions

diff --git a/llvm/examples/OrcV2Examples/CMakeLists.txt b/llvm/examples/OrcV2Examples/CMakeLists.txt
index f1c51b79690a..b6d8d705f216 100644
--- a/llvm/examples/OrcV2Examples/CMakeLists.txt
+++ b/llvm/examples/OrcV2Examples/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_subdirectory(BasicOrcV2CBindings)
 add_subdirectory(LLJITDumpObjects)
-add_subdirectory(LLJITWithObjectCache)
 add_subdirectory(LLJITWithCustomObjectLinkingLayer)
+add_subdirectory(LLJITWithGDBRegistrationListener)
 add_subdirectory(LLJITWithLazyReexports)
+add_subdirectory(LLJITWithObjectCache)
 add_subdirectory(LLJITWithObjectLinkingLayerPlugin)
diff --git a/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/CMakeLists.txt b/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/CMakeLists.txt
new file mode 100644
index 000000000000..5bf96425d60e
--- /dev/null
+++ b/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/CMakeLists.txt
@@ -0,0 +1,16 @@
+set(LLVM_LINK_COMPONENTS
+  Core
+  IRReader
+  JITLink
+  OrcJIT
+  Support
+  nativecodegen
+  )
+
+add_llvm_example(LLJITWithGDBRegistrationListener
+  LLJITWithGDBRegistrationListener.cpp
+  )
+
+# We want JIT'd code to be able to link against process symbols like printf
+# for this example, so make sure they're exported.
+export_executable_symbols(LLJITWithGDBRegistrationListener)
diff --git a/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp b/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
new file mode 100644
index 000000000000..5cf7cd00ffc5
--- /dev/null
+++ b/llvm/examples/OrcV2Examples/LLJITWithGDBRegistrationListener/LLJITWithGDBRegistrationListener.cpp
@@ -0,0 +1,109 @@
+//===--------------- LLJITWithCustomObjectLinkingLayer.cpp ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file shows how to switch LLJIT to use a custom object linking layer (we
+// use ObjectLinkingLayer, which is backed by JITLink, as an example).
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "../ExampleModules.h"
+
+using namespace llvm;
+using namespace llvm::orc;
+
+ExitOnError ExitOnErr;
+
+static cl::opt<std::string>
+    EntryPointName("entry", cl::desc("Symbol to call as main entry point"),
+                   cl::init("main"));
+
+static cl::list<std::string> InputFiles(cl::Positional, cl::OneOrMore,
+                                        cl::desc("input files"));
+
+static cl::list<std::string> InputArgv("args", cl::Positional,
+                                       cl::desc("<program arguments>..."),
+                                       cl::ZeroOrMore, cl::PositionalEatsArgs);
+
+int main(int argc, char *argv[]) {
+  // Initialize LLVM.
+  InitLLVM X(argc, argv);
+
+  InitializeNativeTarget();
+  InitializeNativeTargetAsmPrinter();
+
+  cl::ParseCommandLineOptions(argc, argv, "LLJITWithCustomObjectLinkingLayer");
+  ExitOnErr.setBanner(std::string(argv[0]) + ": ");
+
+  // Detect the host and set code model to small.
+  auto JTMB = ExitOnErr(JITTargetMachineBuilder::detectHost());
+  if (!JTMB.getTargetTriple().isOSLinux())
+    errs()
+        << "Warning: This demo may not work for platforms other than Linux.\n";
+
+  // Create an LLJIT instance and use a custom object linking layer creator to
+  // register the GDBRegistrationListener with our RTDyldObjectLinkingLayer.
+  auto J =
+      ExitOnErr(LLJITBuilder()
+                    .setJITTargetMachineBuilder(std::move(JTMB))
+                    .setObjectLinkingLayerCreator([&](ExecutionSession &ES,
+                                                      const Triple &TT) {
+                      auto GetMemMgr = []() {
+                        return std::make_unique<SectionMemoryManager>();
+                      };
+                      auto ObjLinkingLayer =
+                          std::make_unique<RTDyldObjectLinkingLayer>(
+                              ES, std::move(GetMemMgr));
+                      ObjLinkingLayer->registerJITEventListener(
+                          *JITEventListener::createGDBRegistrationListener());
+                      return ObjLinkingLayer;
+                    })
+                    .create());
+
+  // Make sure that our process symbols are visible to JIT'd code.
+  {
+    MangleAndInterner Mangle(J->getExecutionSession(), J->getDataLayout());
+    J->getMainJITDylib().addGenerator(
+        ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
+            J->getDataLayout().getGlobalPrefix(),
+            [MainName = Mangle("main")](const orc::SymbolStringPtr &Name) {
+              return Name != MainName;
+            })));
+  }
+
+  // Load the input modules.
+  for (auto &InputFile : InputFiles) {
+    auto Ctx = std::make_unique<LLVMContext>();
+    SMDiagnostic Err;
+    std::unique_ptr<Module> M = parseIRFile(InputFile, Err, *Ctx);
+    if (!M) {
+      Err.print(argv[0], errs());
+      return 1;
+    }
+
+    ExitOnErr(J->addIRModule(ThreadSafeModule(std::move(M), std::move(Ctx))));
+  }
+
+  // Look up the entry point, cast it to a C main function pointer, then use
+  // runAsMain to call it.
+  auto EntrySym = ExitOnErr(J->lookup(EntryPointName));
+  auto EntryFn =
+      jitTargetAddressToFunction<int (*)(int, char *[])>(EntrySym.getAddress());
+
+  return runAsMain(EntryFn, InputArgv, StringRef(InputFiles.front()));
+}
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
index 435c882d506d..b6b422002524 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
@@ -115,15 +116,23 @@ public:
     return *this;
   }
 
+  /// Register a JITEventListener.
+  void registerJITEventListener(JITEventListener &L);
+
+  /// Unregister a JITEventListener.
+  void unregisterJITEventListener(JITEventListener &L);
+
 private:
   Error onObjLoad(VModuleKey K, MaterializationResponsibility &R,
-                  object::ObjectFile &Obj,
+                  object::ObjectFile &Obj, RuntimeDyld::MemoryManager *MemMgr,
                   std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo,
                   std::map<StringRef, JITEvaluatedSymbol> Resolved,
                   std::set<StringRef> &InternalSymbols);
 
-  void onObjEmit(VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer,
-                 MaterializationResponsibility &R, Error Err);
+  void onObjEmit(VModuleKey K, MaterializationResponsibility &R,
+                 object::ObjectFile &Obj,
+                 std::unique_ptr<MemoryBuffer> ObjBuffer,
+                 RuntimeDyld::MemoryManager *MemMgr, Error Err);
 
   mutable std::mutex RTDyldLayerMutex;
   GetMemoryManagerFunction GetMemoryManager;
@@ -133,6 +142,10 @@ private:
   bool OverrideObjectFlags = false;
   bool AutoClaimObjectSymbols = false;
   std::vector<std::unique_ptr<RuntimeDyld::MemoryManager>> MemMgrs;
+  std::vector<JITEventListener *> EventListeners;
+  DenseMap<RuntimeDyld::MemoryManager *,
+           std::unique_ptr<RuntimeDyld::LoadedObjectInfo>>
+      LoadedObjInfos;
 };
 
 class LegacyRTDyldObjectLinkingLayerBase {
diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index f51bd9d3b1c2..3fafe4a9b6eb 100644
--- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -81,8 +81,12 @@ RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer(
 
 RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() {
   std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
-  for (auto &MemMgr : MemMgrs)
+  for (auto &MemMgr : MemMgrs) {
+    for (auto *L : EventListeners)
+      L->notifyFreeingObject(
+          static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr.get())));
     MemMgr->deregisterEHFrames();
+  }
 }
 
 void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R,
@@ -155,19 +159,35 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R,
 
   jitLinkForORC(
       **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections,
-      [this, K, SharedR, &Obj, InternalSymbols](
+      [this, K, SharedR, &Obj, MemMgr, InternalSymbols](
           std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo,
           std::map<StringRef, JITEvaluatedSymbol> ResolvedSymbols) {
-        return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo),
+        return onObjLoad(K, *SharedR, **Obj, MemMgr, std::move(LoadedObjInfo),
                          ResolvedSymbols, *InternalSymbols);
       },
-      [this, K, SharedR, O = std::move(O)](Error Err) mutable {
-        onObjEmit(K, std::move(O), *SharedR, std::move(Err));
+      [this, K, SharedR, &Obj, O = std::move(O), MemMgr](Error Err) mutable {
+        onObjEmit(K, *SharedR, **Obj, std::move(O), MemMgr, std::move(Err));
       });
 }
 
+void RTDyldObjectLinkingLayer::registerJITEventListener(JITEventListener &L) {
+  std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+  assert(llvm::none_of(EventListeners,
+                       [&](JITEventListener *O) { return O == &L; }) &&
+         "Listener has already been registered");
+  EventListeners.push_back(&L);
+}
+
+void RTDyldObjectLinkingLayer::unregisterJITEventListener(JITEventListener &L) {
+  std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+  auto I = llvm::find(EventListeners, &L);
+  assert(I != EventListeners.end() && "Listener not registered");
+  EventListeners.erase(I);
+}
+
 Error RTDyldObjectLinkingLayer::onObjLoad(
     VModuleKey K, MaterializationResponsibility &R, object::ObjectFile &Obj,
+    RuntimeDyld::MemoryManager *MemMgr,
     std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo,
     std::map<StringRef, JITEvaluatedSymbol> Resolved,
     std::set<StringRef> &InternalSymbols) {
@@ -252,12 +272,17 @@ Error RTDyldObjectLinkingLayer::onObjLoad(
   if (NotifyLoaded)
     NotifyLoaded(K, Obj, *LoadedObjInfo);
 
+  std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+  assert(!LoadedObjInfos.count(MemMgr) && "Duplicate loaded info for MemMgr");
+  LoadedObjInfos[MemMgr] = std::move(LoadedObjInfo);
+
   return Error::success();
 }
 
 void RTDyldObjectLinkingLayer::onObjEmit(
-    VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer,
-    MaterializationResponsibility &R, Error Err) {
+    VModuleKey K, MaterializationResponsibility &R, object::ObjectFile &Obj,
+    std::unique_ptr<MemoryBuffer> ObjBuffer, RuntimeDyld::MemoryManager *MemMgr,
+    Error Err) {
   if (Err) {
     getExecutionSession().reportError(std::move(Err));
     R.failMaterialization();
@@ -272,6 +297,16 @@ void RTDyldObjectLinkingLayer::onObjEmit(
 
   if (NotifyEmitted)
     NotifyEmitted(K, std::move(ObjBuffer));
+
+  // Run EventListener notifyLoaded callbacks.
+  std::lock_guard<std::mutex> Lock(RTDyldLayerMutex);
+  auto LOIItr = LoadedObjInfos.find(MemMgr);
+  assert(LOIItr != LoadedObjInfos.end() && "LoadedObjInfo missing");
+  for (auto *L : EventListeners)
+    L->notifyObjectLoaded(
+        static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr)), Obj,
+        *LOIItr->second);
+  LoadedObjInfos.erase(MemMgr);
 }
 
 LegacyRTDyldObjectLinkingLayer::LegacyRTDyldObjectLinkingLayer(
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index 80e08f0a94ff..87d51426bc65 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -30,6 +30,7 @@
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
 #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h"
+#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/OrcMCJITReplacement.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/IRBuilder.h"
@@ -892,6 +893,11 @@ int runOrcLazyJIT(const char *ProgName) {
 
   auto J = ExitOnErr(Builder.create());
 
+  if (TT->isOSBinFormatELF())
+    static_cast<llvm::orc::RTDyldObjectLinkingLayer &>(J->getObjLinkingLayer())
+        .registerJITEventListener(
+            *JITEventListener::createGDBRegistrationListener());
+
   if (PerModuleLazy)
     J->setPartitionFunction(orc::CompileOnDemandLayer::compileWholeModule);