[ESI Runtime] Pluggable channel engines

include/circt/Dialect/ESI/ESIInterfaces.td

@@ -64,7 +64,7 @@ def IsManifestData : OpInterface<"IsManifestData"> {
   }];
 
   let methods = [
-    StaticInterfaceMethod<
+    InterfaceMethod<
       "Get the class name for this op.",
       "StringRef", "getManifestClass", (ins)
     >,

include/circt/Dialect/ESI/ESIManifest.td

@@ -99,14 +99,15 @@ def ServiceImplRecordOp : ESI_Op<"manifest.service_impl", [
   }];
 
   let arguments = (ins AppIDAttr:$appID,
+                       DefaultValuedAttr<UnitAttr, "false">:$isEngine,
                        OptionalAttr<FlatSymbolRefAttr>:$service,
                        OptionalAttr<StrAttr>:$stdService,
                        StrAttr:$serviceImplName,
                        DictionaryAttr:$implDetails);
   let regions = (region SizedRegion<1>:$reqDetails);
   let assemblyFormat = [{
     qualified($appID) (`svc` $service^)? (`std` $stdService^)?
-   `by` $serviceImplName `with` $implDetails
+   `by` $serviceImplName (`engine` $isEngine^)? `with` $implDetails
     attr-dict-with-keyword custom<ServiceImplRecordReqDetails>($reqDetails)
   }];
 

integration_test/Dialect/ESI/runtime/loopback.mlir

@@ -143,8 +143,6 @@ hw.module @top(in %clk: !seq.clock, in %rst: i1) {
 // QUERY-HIER: * Instance:top
 // QUERY-HIER: * Ports:
 // QUERY-HIER:     internal_write:
-// QUERY-HIER:       ack: !esi.channel<i0>
-// QUERY-HIER:       req: !esi.channel<!hw.struct<address: i5, data: i64>>
 // QUERY-HIER:     func1: function i16(i16)
 // QUERY-HIER:     structFunc: function !hw.struct<x: si8, y: si8>(!hw.struct<a: ui16, b: si8>)
 // QUERY-HIER:     arrayFunc: function !hw.array<2xsi8>(!hw.array<1xsi8>)

lib/Dialect/ESI/ESIOps.cpp

@@ -768,7 +768,9 @@ void ESIPureModuleOp::setHWModuleType(hw::ModuleType type) {
 // Manifest ops.
 //===----------------------------------------------------------------------===//
 
-StringRef ServiceImplRecordOp::getManifestClass() { return "service"; }
+StringRef ServiceImplRecordOp::getManifestClass() {
+  return getIsEngine() ? "engine" : "service";
+}
 
 void ServiceImplRecordOp::getDetails(SmallVectorImpl<NamedAttribute> &results) {
   auto *ctxt = getContext();

lib/Dialect/ESI/ESIServices.cpp

@@ -58,6 +58,7 @@ instantiateCosimEndpointOps(ServiceImplementReqOp implReq,
   }
 
   Block &connImplBlock = implRecord.getReqDetails().front();
+  implRecord.setIsEngine(true);
   OpBuilder implRecords = OpBuilder::atBlockEnd(&connImplBlock);
 
   // Assemble the name to use for an endpoint.
@@ -300,8 +301,9 @@ ServiceGeneratorDispatcher::generate(ServiceImplementReqOp req,
   // the generator for possible modification.
   OpBuilder b(req);
   auto implRecord = b.create<ServiceImplRecordOp>(
-      req.getLoc(), req.getAppID(), req.getServiceSymbolAttr(),
-      req.getStdServiceAttr(), req.getImplTypeAttr(), b.getDictionaryAttr({}));
+      req.getLoc(), req.getAppID(), /*isEngine=*/false,
+      req.getServiceSymbolAttr(), req.getStdServiceAttr(),
+      req.getImplTypeAttr(), b.getDictionaryAttr({}));
   implRecord.getReqDetails().emplaceBlock();
 
   return genF->second(req, decl, implRecord);

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -100,6 +100,7 @@ set(ESICppRuntimeSources
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Context.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Common.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Design.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Engines.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Logging.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Manifest.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/cpp/lib/Services.cpp

lib/Dialect/ESI/runtime/cpp/include/esi/Accelerator.h

@@ -23,6 +23,7 @@
 
 #include "esi/Context.h"
 #include "esi/Design.h"
+#include "esi/Engines.h"
 #include "esi/Manifest.h"
 #include "esi/Ports.h"
 #include "esi/Services.h"
@@ -92,11 +93,6 @@ class AcceleratorConnection {
   // each level of the tree.
   using ServiceTable = std::map<std::string, services::Service *>;
 
-  /// Request the host side channel ports for a particular instance (identified
-  /// by the AppID path). For convenience, provide the bundle type.
-  virtual std::map<std::string, ChannelPort &>
-  requestChannelsFor(AppIDPath, const BundleType *, const ServiceTable &) = 0;
-
   /// Return a pointer to the accelerator 'service' thread (or threads). If the
   /// thread(s) are not running, they will be started when this method is
   /// called. `std::thread` is used. If users don't want the runtime to spin up
@@ -126,7 +122,23 @@ class AcceleratorConnection {
   /// accelerator to this connection. Returns a raw pointer to the object.
   Accelerator *takeOwnership(std::unique_ptr<Accelerator> accel);
 
+  /// Create a new engine for channel communication with the accelerator. The
+  /// default is to call the global `createEngine` to get an engine which has
+  /// registered itself. Individual accelerator connection backends can override
+  /// this to customize behavior.
+  virtual void createEngine(const std::string &engineTypeName, AppIDPath idPath,
+                            const ServiceImplDetails &details,
+                            const HWClientDetails &clients);
+  virtual const BundleEngineMap &getEngineMapFor(AppIDPath id) {
+    return clientEngines[id];
+  }
+
 protected:
+  /// If `createEngine` is overridden, this method should be called to register
+  /// the engine and all of the channels it services.
+  void registerEngine(AppIDPath idPath, std::unique_ptr<Engine> engine,
+                      const HWClientDetails &clients);
+
   /// Called by `getServiceImpl` exclusively. It wraps the pointer returned by
   /// this in a unique_ptr and caches it. Separate this from the
   /// wrapping/caching since wrapping/caching is an implementation detail.
@@ -135,6 +147,11 @@ class AcceleratorConnection {
                                  const ServiceImplDetails &details,
                                  const HWClientDetails &clients) = 0;
 
+  /// Collection of owned engines.
+  std::map<AppIDPath, std::unique_ptr<Engine>> ownedEngines;
+  /// Mapping of clients to their servicing engines.
+  std::map<AppIDPath, BundleEngineMap> clientEngines;
+
 private:
   /// ESI accelerator context.
   Context &ctxt;

lib/Dialect/ESI/runtime/cpp/include/esi/Engines.h

@@ -0,0 +1,117 @@
+//===- Engines.h - Implement port communication -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// DO NOT EDIT!
+// This file is distributed as part of an ESI package. The source for this file
+// should always be modified within CIRCT.
+//
+//===----------------------------------------------------------------------===//
+//
+// Engines (as in DMA engine) implement the actual communication between the
+// host and the accelerator. They are low level of the ESI runtime API and are
+// not intended to be used directly by users.
+//
+// They are called "engines" rather than "DMA engines" since communication need
+// not be implemented via DMA.
+//
+//===----------------------------------------------------------------------===//
+
+// NOLINTNEXTLINE(llvm-header-guard)
+#ifndef ESI_ENGINGES_H
+#define ESI_ENGINGES_H
+
+#include "esi/Common.h"
+#include "esi/Ports.h"
+#include "esi/Services.h"
+#include "esi/Utils.h"
+
+#include <cassert>
+#include <future>
+
+namespace esi {
+
+/// Engines implement the actual channel communication between the host and the
+/// accelerator. Engines can support multiple channels. They are low level of
+/// the ESI runtime API and are not intended to be used directly by users.
+class Engine {
+public:
+  virtual ~Engine() = default;
+  /// Start the engine, if applicable.
+  virtual void connect(){};
+  /// Stop the engine, if applicable.
+  virtual void disconnect(){};
+  /// Get a port for a channel, from the cache if it exists or create it. An
+  /// engine may override this method if different behavior is desired.
+  virtual ChannelPort &requestPort(AppIDPath idPath,
+                                   const std::string &channelName,
+                                   BundleType::Direction dir, const Type *type);
+
+protected:
+  /// Each engine needs to know how to create a ports. This method is called if
+  /// a port doesn't exist in the engine cache.
+  virtual std::unique_ptr<ChannelPort>
+  createPort(AppIDPath idPath, const std::string &channelName,
+             BundleType::Direction dir, const Type *type) = 0;
+
+private:
+  std::map<std::pair<AppIDPath, std::string>, std::unique_ptr<ChannelPort>>
+      ownedPorts;
+};
+
+/// Since engines can support multiple channels BUT not necessarily all of the
+/// channels in a bundle, a mapping from bundle channels to engines is needed.
+class BundleEngineMap {
+  friend class AcceleratorConnection;
+
+public:
+  /// Request ports for all the channels in a bundle. If the engine doesn't
+  /// exist for a particular channel, skip said channel.
+  PortMap requestPorts(const AppIDPath &idPath,
+                       const BundleType *bundleType) const;
+
+private:
+  /// Set a particlar engine for a particular channel. Should only be called by
+  /// AcceleratorConnection while registering engines.
+  void setEngine(const std::string &channelName, Engine *engine);
+  std::map<std::string, Engine *> bundleEngineMap;
+};
+
+namespace registry {
+
+/// Create an engine by name. This is the primary way to create engines for
+/// "normal" backends.
+std::unique_ptr<Engine> createEngine(AcceleratorConnection &conn,
+                                     const std::string &dmaEngineName,
+                                     AppIDPath idPath,
+                                     const ServiceImplDetails &details,
+                                     const HWClientDetails &clients);
+
+namespace internal {
+
+/// Engines can register themselves for pluggable functionality.
+using EngineCreate = std::function<std::unique_ptr<Engine>(
+    AcceleratorConnection &conn, AppIDPath idPath,
+    const ServiceImplDetails &details, const HWClientDetails &clients)>;
+void registerEngine(const std::string &name, EngineCreate create);
+
+/// Helper struct to register engines.
+template <typename TEngine>
+struct RegisterEngine {
+  RegisterEngine(const char *name) { registerEngine(name, &TEngine::create); }
+};
+
+#define REGISTER_ENGINE(Name, TEngine)                                         \
+  static ::esi::registry::internal::RegisterEngine<TEngine>                    \
+  __register_engine____LINE__(Name)
+
+} // namespace internal
+} // namespace registry
+
+} // namespace esi
+
+#endif // ESI_PORTS_H

lib/Dialect/ESI/runtime/cpp/include/esi/Ports.h

@@ -25,6 +25,9 @@
 
 namespace esi {
 
+class ChannelPort;
+using PortMap = std::map<std::string, ChannelPort &>;
+
 /// Unidirectional channels are the basic communication primitive between the
 /// host and accelerator. A 'ChannelPort' is the host side of a channel. It can
 /// be either read or write but not both. At this level, channels are untyped --
@@ -190,7 +193,7 @@ class BundlePort {
   }
 
   /// Construct a port.
-  BundlePort(AppID id, std::map<std::string, ChannelPort &> channels);
+  BundlePort(AppID id, const BundleType *type, PortMap channels);
   virtual ~BundlePort() = default;
 
   /// Get the ID of the port.
@@ -202,9 +205,7 @@ class BundlePort {
   /// ordinary users should not use. You have been warned.
   WriteChannelPort &getRawWrite(const std::string &name) const;
   ReadChannelPort &getRawRead(const std::string &name) const;
-  const std::map<std::string, ChannelPort &> &getChannels() const {
-    return channels;
-  }
+  const PortMap &getChannels() const { return channels; }
 
   /// Cast this Bundle port to a subclass which is actually useful. Returns
   /// nullptr if the cast fails.
@@ -224,9 +225,10 @@ class BundlePort {
     return result;
   }
 
-private:
+protected:
   AppID id;
-  std::map<std::string, ChannelPort &> channels;
+  const BundleType *type;
+  PortMap channels;
 };
 
 } // namespace esi