Add support for data library referencing (AcademySoftwareFoundation#2054

)

### Background
MaterialX documents are required to be complete self contained with all required node definitions. This is done by using the `mx::importLibrary` API. While processing large number of documents the Import and Merge workflows can cause performance and memory challenges.

### Proposal 
Since the MaterialX library or the Data library hosts definitions, we propose that these definitions could be referenced by a document instead of importing them. This concept of referencing the Data library may have some workflow impact such as, do new definitions to into Data library or the document or can a document have local definitions? The answer depends on the specific artist or studio workflow.

To maintain compatibility with existing usage patterns, we have introduced a new set of methods **that allow registration of data library for a document by setting a document reference to it**.
1. setDataLibrary(ConstDocumentPtr dataLibrary);
2. getDataLibrary()
3. hasDataLibrary()

### Using this new Data library registration, two usage patterns are possible
**Build library and import**
This is the current usage pattern
- use `importlibrary `to build your datalibrary
- use `importLibrary `to import the datalibrary to the working document.

**Build library and register**
Proposed usage pattern
- use `importlibrary` to build your datalibrary
- use `setDataLibrary `to assign the datalibrary to the working document.
This can be extended were 3rd party defs can be imported into the datalibrary

**Other changes include**
- Updates to `getChildOfType` and `getChildrenOfType` to support Data Library.
- Updates to redirect _Document::get*_ calls to process local document content and Data Library.

source/JsMaterialX/JsMaterialXCore/JsDocument.cpp

@@ -23,6 +23,9 @@ EMSCRIPTEN_BINDINGS(document)
         .class_function("createDocument", &mx::Document::createDocument<mx::Document>)
         .function("initialize", &mx::Document::initialize)
         .function("copy", &mx::Document::copy)
+        .function("setDataLibrary", &mx::Document::setDataLibrary)
+        .function("getDataLibrary", &mx::Document::getDataLibrary)
+        .function("hasDataLibrary", &mx::Document::hasDataLibrary)
         .function("importLibrary", &mx::Document::importLibrary)
         .function("getReferencedSourceUris", ems::optional_override([](mx::Document &self) {
             mx::StringSet set = self.getReferencedSourceUris();

source/MaterialXCore/Document.cpp

@@ -357,34 +357,40 @@ vector<OutputPtr> Document::getMaterialOutputs() const
 
 vector<NodeDefPtr> Document::getMatchingNodeDefs(const string& nodeName) const
 {
+    // Recurse to data library if present.
+    vector<NodeDefPtr> matchingNodeDefs = hasDataLibrary() ? 
+                                          getDataLibrary()->getMatchingNodeDefs(nodeName) :
+                                          vector<NodeDefPtr>();
+
     // Refresh the cache.
     _cache->refresh();
 
     // Return all nodedefs matching the given node name.
     if (_cache->nodeDefMap.count(nodeName))
     {
-        return _cache->nodeDefMap.at(nodeName);
-    }
-    else
-    {
-        return vector<NodeDefPtr>();
+        matchingNodeDefs.insert(matchingNodeDefs.end(), _cache->nodeDefMap.at(nodeName).begin(), _cache->nodeDefMap.at(nodeName).end());
     }
+
+    return matchingNodeDefs;
 }
 
 vector<InterfaceElementPtr> Document::getMatchingImplementations(const string& nodeDef) const
 {
+    // Recurse to data library if present.
+    vector<InterfaceElementPtr> matchingImplementations = hasDataLibrary() ?
+                                                          getDataLibrary()->getMatchingImplementations(nodeDef) :
+                                                          vector<InterfaceElementPtr>();
+
     // Refresh the cache.
     _cache->refresh();
 
     // Return all implementations matching the given nodedef string.
     if (_cache->implementationMap.count(nodeDef))
     {
-        return _cache->implementationMap.at(nodeDef);
-    }
-    else
-    {
-        return vector<InterfaceElementPtr>();
+        matchingImplementations.insert(matchingImplementations.end(), _cache->implementationMap.at(nodeDef).begin(), _cache->implementationMap.at(nodeDef).end());
     }
+
+    return matchingImplementations;
 }
 
 bool Document::validate(string* message) const

source/MaterialXCore/Document.h

@@ -50,18 +50,41 @@ class MX_CORE_API Document : public GraphElement
     {
         DocumentPtr doc = createDocument<Document>();
         doc->copyContentFrom(getSelf());
+        doc->setDataLibrary(getDataLibrary());
         return doc;
     }
 
-    /// Import the given document as a library within this document.
-    /// The contents of the library document are copied into this one, and
+    /// Get a list of source URIs referenced by the document
+    StringSet getReferencedSourceUris() const;
+
+    /// @name Data Libraries
+    /// @{
+
+    /// Store a reference to a data library in this document.
+    void setDataLibrary(ConstDocumentPtr dataLibrary)
+    {
+        _dataLibrary = dataLibrary;
+    }
+
+    /// Return true if this document has a data library.
+    bool hasDataLibrary() const
+    {
+        return (_dataLibrary != nullptr);
+    }
+
+    /// Return the data library, if any, referenced by this document.
+    ConstDocumentPtr getDataLibrary() const
+    {
+        return _dataLibrary;
+    }
+
+    /// Import the given data library into this document.
+    /// The contents of the data library are copied into this one, and
     /// are assigned the source URI of the library.
-    /// @param library The library document to be imported.
+    /// @param library The data library to be imported.
     void importLibrary(const ConstDocumentPtr& library);
 
-    /// Get a list of source URI's referenced by the document
-    StringSet getReferencedSourceUris() const;
-
+    /// @}
     /// @name NodeGraph Elements
     /// @{
 
@@ -679,6 +702,9 @@ class MX_CORE_API Document : public GraphElement
 
   private:
     class Cache;
+
+  private:
+    ConstDocumentPtr _dataLibrary;
     std::unique_ptr<Cache> _cache;
 };
 

source/MaterialXCore/Element.cpp

@@ -286,13 +286,27 @@ ElementPtr Element::changeChildCategory(ElementPtr child, const string& category
 
 template <class T> shared_ptr<T> Element::getChildOfType(const string& name) const
 {
-    ElementPtr child = getChild(name);
+    ElementPtr child;
+    ConstDocumentPtr doc = asA<Document>();
+    if (doc && doc->hasDataLibrary())
+    {
+        child = doc->getDataLibrary()->getChild(name);
+    }
+    if (!child)
+    {
+        child = getChild(name);
+    }
     return child ? child->asA<T>() : shared_ptr<T>();
 }
 
 template <class T> vector<shared_ptr<T>> Element::getChildrenOfType(const string& category) const
 {
     vector<shared_ptr<T>> children;
+    ConstDocumentPtr doc = asA<Document>();
+    if (doc && doc->hasDataLibrary())
+    {
+        children = doc->getDataLibrary()->getChildrenOfType<T>(category);
+    }
     for (ElementPtr child : _childOrder)
     {
         shared_ptr<T> instance = child->asA<T>();

source/MaterialXTest/MaterialXCore/Document.cpp

@@ -106,7 +106,11 @@ TEST_CASE("Document", "[document]")
     REQUIRE(customLibrary->validate());
 
     // Import the custom library.
-    doc->importLibrary(customLibrary);
+    mx::DocumentPtr customDatalibrary = mx::createDocument();
+    customDatalibrary->importLibrary(customLibrary);
+
+    // Set data library
+    doc->setDataLibrary(customDatalibrary);
     mx::NodeGraphPtr importedNodeGraph = doc->getNodeGraph("custom:NG_custom");
     mx::NodeDefPtr importedNodeDef = doc->getNodeDef("custom:ND_simpleSrf");
     mx::ImplementationPtr importedImpl = doc->getImplementation("custom:IM_custom");

source/MaterialXTest/MaterialXCore/Node.cpp

@@ -672,7 +672,7 @@ TEST_CASE("Node Definition Creation", "[nodedef]")
 
     mx::DocumentPtr doc = mx::createDocument();
     mx::readFromXmlFile(doc, "resources/Materials/TestSuite/stdlib/definition/definition_from_nodegraph.mtlx", searchPath);
-    doc->importLibrary(stdlib);
+    doc->setDataLibrary(stdlib);
 
     mx::NodeGraphPtr graph = doc->getNodeGraph("test_colorcorrect");
     REQUIRE(graph);

source/MaterialXTest/MaterialXGenShader/GenShader.cpp

@@ -161,7 +161,7 @@ TEST_CASE("GenShader: Transparency Regression Check", "[genshader]")
         bool testValue = transparencyTest[i];
 
         mx::DocumentPtr testDoc = mx::createDocument();
-        testDoc->importLibrary(libraries);
+        testDoc->setDataLibrary(libraries);
 
         try
         {
@@ -207,7 +207,7 @@ void testDeterministicGeneration(mx::DocumentPtr libraries, mx::GenContext& cont
     {
         mx::DocumentPtr testDoc = mx::createDocument();
         mx::readFromXmlFile(testDoc, testFile);
-        testDoc->importLibrary(libraries);
+        testDoc->setDataLibrary(libraries);
 
         // Keep the document alive to make sure
         // new memory is allocated for each run
@@ -272,7 +272,7 @@ void checkPixelDependencies(mx::DocumentPtr libraries, mx::GenContext& context)
 
     mx::DocumentPtr testDoc = mx::createDocument();
     mx::readFromXmlFile(testDoc, testFile);
-    testDoc->importLibrary(libraries);
+    testDoc->setDataLibrary(libraries);
 
     mx::ElementPtr element = testDoc->getChild(testElement);
     CHECK(element);
@@ -385,7 +385,7 @@ TEST_CASE("GenShader: Track Application Variables", "[genshader]")
 
     mx::DocumentPtr testDoc = mx::createDocument();
     mx::readFromXmlString(testDoc, testDocumentString);
-    testDoc->importLibrary(libraries);
+    testDoc->setDataLibrary(libraries);
 
     mx::ElementPtr element = testDoc->getChild(testElement);
     CHECK(element);

source/MaterialXTest/MaterialXGenShader/GenShaderUtil.cpp

@@ -372,7 +372,7 @@ void shaderGenPerformanceTest(mx::GenContext& context)
     std::shuffle(loadedDocuments.begin(), loadedDocuments.end(), rng);
     for (const auto& doc : loadedDocuments)
     {
-        doc->importLibrary(nodeLibrary);
+        doc->setDataLibrary(nodeLibrary);
         std::vector<mx::TypedElementPtr> elements = mx::findRenderableElements(doc);
 
         REQUIRE(elements.size() > 0);
@@ -721,7 +721,7 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons
         bool importedLibrary = false;
         try
         {
-            doc->importLibrary(_dependLib);
+            doc->setDataLibrary(_dependLib);
             importedLibrary = true;
         }
         catch (mx::Exception& e)
@@ -734,8 +734,8 @@ void ShaderGeneratorTester::validate(const mx::GenOptions& generateOptions, cons
         }
 
         // Find and register lights
-        findLights(doc, _lights);
-        registerLights(doc, _lights, context);
+        findLights(_dependLib, _lights);
+        registerLights(_dependLib, _lights, context);
 
         // Find elements to render in the document
         std::vector<mx::TypedElementPtr> elements;

source/MaterialXTest/MaterialXRender/RenderUtil.cpp

@@ -223,7 +223,7 @@ bool ShaderRenderTester::validate(const mx::FilePath optionsFilePath)
             // colliding with implementations in previous test cases.
             context.clearNodeImplementations();
 
-            doc->importLibrary(dependLib);
+            doc->setDataLibrary(dependLib);
             ioTimer.endTimer();
 
             validateTimer.startTimer();

source/MaterialXView/Viewer.cpp

@@ -1316,8 +1316,8 @@ void Viewer::loadDocument(const mx::FilePath& filename, mx::DocumentPtr librarie
         mx::readFromXmlFile(doc, filename, _searchPath, &readOptions);
         _materialSearchPath = mx::getSourceSearchPath(doc);
 
-        // Import libraries.
-        doc->importLibrary(libraries);
+        // Store data library reference.
+        doc->setDataLibrary(libraries);
 
         // Apply direct lights.
         applyDirectLights(doc);

source/PyMaterialX/PyMaterialXCore/PyDocument.cpp

@@ -29,6 +29,9 @@ void bindPyDocument(py::module& mod)
     py::class_<mx::Document, mx::DocumentPtr, mx::GraphElement>(mod, "Document")
         .def("initialize", &mx::Document::initialize)
         .def("copy", &mx::Document::copy)
+        .def("setDataLibrary", &mx::Document::setDataLibrary)
+        .def("getDataLibrary", &mx::Document::getDataLibrary)
+        .def("hasDataLibrary", &mx::Document::hasDataLibrary)
         .def("importLibrary", &mx::Document::importLibrary)
         .def("getReferencedSourceUris", &mx::Document::getReferencedSourceUris)
         .def("addNodeGraph", &mx::Document::addNodeGraph,