Don't merge a layer if it's an entry layer (#1103)

Don't merge a layer if it's an entry layer
Add test case for merging duplicate layers that are also entry layers.

Remove DASSERT that aborted entry layers being merged

CMakeLists.txt

@@ -285,6 +285,7 @@ TESTSUITE ( aastep allowconnect-err and-or-not-synonyms arithmetic
             layers-nonlazycopy layers-repeatedoutputs
             linearstep
             logic loop matrix message
+            mergeinstances-duplicate-entrylayers
             mergeinstances-nouserdata mergeinstances-vararray
             metadata-braces miscmath missing-shader
             noise noise-cell

src/liboslexec/context.cpp

@@ -136,7 +136,8 @@ ShadingContext::execute_init (ShaderGroup &sgroup, ShaderGlobals &ssg, bool run)
 bool
 ShadingContext::execute_layer (ShaderGlobals &ssg, int layernumber)
 {
-    DASSERT (group() && group()->nlayers() && !group()->does_nothing());
+    if (!group() || group()->nlayers() == 0 || group()->does_nothing())
+        return false;
     DASSERT (ssg.context == this && ssg.renderer == renderer());
 
     int profile = shadingsys().m_profile;

src/liboslexec/shadingsys.cpp

@@ -2995,8 +2995,8 @@ ShadingSystemImpl::merge_instances (ShaderGroup &group, bool post_opt)
 
     // Loop over all layers...
     for (int a = 0;  a < nlayers-1;  ++a) {
-        if (group[a]->unused())    // Don't merge a layer that's not used
-            continue;
+        if (group[a]->unused() || group[a]->entry_layer()) // Don't merge a layer that's not used
+            continue;                                      // or if it's an entry layer
         // Check all later layers...
         for (int b = a+1;  b < nlayers;  ++b) {
             if (group[b]->unused())    // Don't merge a layer that's not used

testsuite/mergeinstances-duplicate-entrylayers/a.osl

@@ -0,0 +1,12 @@
+shader a
+(
+    float frequency = 1,
+    string noiseType = "perlin",
+    color amplitude = 1,
+    int seed = 0,
+    output color out = 0
+)
+{
+    point domain = point(u + seed, v + seed, 0) * frequency;
+    out = noise(noiseType, domain[0], domain[1]) * amplitude;
+}

testsuite/mergeinstances-duplicate-entrylayers/b.osl

@@ -0,0 +1,8 @@
+shader b
+(
+    color in  = 0,
+    output color out = 0
+)
+{
+    out = in;
+}

testsuite/mergeinstances-duplicate-entrylayers/ref/out.txt

@@ -0,0 +1,4 @@
+Compiled a.osl -> a.oso
+Compiled b.osl -> b.oso
+Entry layers: b1(1) b2(3) b3(5)
+

testsuite/mergeinstances-duplicate-entrylayers/run.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+command = testshade("--group data/shadergroup -O2 " +
+	                "--entry b1 --entry b2 --entry b3")

testsuite/mergeinstances-duplicate-entrylayers/shadergroup

@@ -0,0 +1,11 @@
+shader a a1 ;
+shader b b1 ;
+connect a1.out b1.in ;
+
+shader a a2 ;
+shader b b2 ;
+connect a2.out b2.in ;
+
+shader a a3 ;
+shader b b3 ;
+connect a3.out b3.in ;