diff --git a/python/neuroglancer/__init__.py b/python/neuroglancer/__init__.py index f8953d5b0..edafa519b 100644 --- a/python/neuroglancer/__init__.py +++ b/python/neuroglancer/__init__.py @@ -49,7 +49,7 @@ PlaceEllipsoidTool, # noqa: F401 BlendTool, # noqa: F401 OpacityTool, # noqa: F401 - VolumeRenderingModeTool, # noqa: F401 + VolumeRenderingTool, # noqa: F401 VolumeRenderingGainTool, # noqa: F401 VolumeRenderingDepthSamplesTool, # noqa: F401 CrossSectionRenderScaleTool, # noqa: F401 diff --git a/python/neuroglancer/viewer_state.py b/python/neuroglancer/viewer_state.py index 532b15df1..a2b14edba 100644 --- a/python/neuroglancer/viewer_state.py +++ b/python/neuroglancer/viewer_state.py @@ -165,9 +165,9 @@ class OpacityTool(Tool): @export_tool -class VolumeRenderingModeTool(Tool): +class VolumeRenderingTool(Tool): __slots__ = () - TOOL_TYPE = "volumeRenderingMode" + TOOL_TYPE = "volumeRendering" @export_tool diff --git a/python/tests/gain_test.py b/python/tests/gain_test.py new file mode 100644 index 000000000..8b6d795c4 --- /dev/null +++ b/python/tests/gain_test.py @@ -0,0 +1,140 @@ +# @license +# Copyright 2020 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests basic screenshot functionality.""" + +import neuroglancer +import numpy as np +from time import sleep +import pytest +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.common.by import By +from PIL import Image +import io + + + +def add_render_panel(side="left", row=0, col=0): + return neuroglancer.LayerSidePanelState( + side=side, + col=col, + row=row, + tab="rendering", + tabs=["rendering", "source"], + ) + +def add_image_layer(state, **kwargs): + shape = (50,) * 3 + data = np.full(shape=shape, fill_value=255, dtype=np.uint8) + dimensions = neuroglancer.CoordinateSpace( + names=["x", "y", "z"], units="nm", scales=[400, 400, 400] + ) + local_volume = neuroglancer.LocalVolume(data, dimensions) + state.layers["image"] = neuroglancer.ImageLayer( + source=local_volume, + volume_rendering=True, + tool_bindings={ + "A": neuroglancer.VolumeRenderingGainTool(), + }, + panels=[add_render_panel()], + **kwargs, + ) + # state.layout = "3d" + +def get_shader(): + return """ +void main() { + emitRGBA(vec4(1.0, 1.0, 1.0, 0.001)); + } + """ + +@pytest.fixture() +def shared_webdriver(request, webdriver): + gainValue = request.node.get_closest_marker("gain_value").args[0] + with webdriver.viewer.txn() as s: + add_image_layer(s, shader=get_shader()) + s.layers["image"].volumeRenderingGain = gainValue + yield webdriver + +no_gain_screenshot = None +gain_screenshot = None + +@pytest.mark.timeout(600) +@pytest.mark.gain_value(10) +def test_gain(shared_webdriver): + global gain_screenshot + global gain_avg + shared_webdriver.sync() + sleep(2) + WebDriverWait(shared_webdriver.driver, 60).until( + lambda driver: driver.execute_script('return document.readyState') == 'complete' + ) + print("Layer loaded") + canvas_element = WebDriverWait(shared_webdriver.driver, 10).until( + EC.presence_of_element_located((By.CLASS_NAME, 'neuroglancer-layer-group-viewer')) + ) + screenshot = canvas_element.screenshot_as_png + with open('gain_screenshot.png', 'wb') as file: + file.write(screenshot) + print("Screenshot taken") + # Convert the screenshot to a NumPy array + image = Image.open(io.BytesIO(screenshot)) + gain_screenshot = np.array(image) + assert gain_screenshot.size != 0, "Image is empty" + # Check if the image contains valid pixel values + assert np.all(gain_screenshot >= 0) and np.all(gain_screenshot <= 255), "Image contains invalid pixel values" + gain_avg = np.mean(gain_screenshot) + print('Gain average pixel value:') + print(gain_avg) + +@pytest.mark.timeout(600) +@pytest.mark.gain_value(0) +def test_no_gain(shared_webdriver): + + global no_gain_avg + global no_gain_screenshot + + # shared_webdriver.sync() + sleep(2) + WebDriverWait(shared_webdriver.driver, 60).until( + lambda driver: driver.execute_script('return document.readyState') == 'complete' + ) + + print("Layer loaded") + canvas_element = WebDriverWait(shared_webdriver.driver, 10).until( + EC.presence_of_element_located((By.CLASS_NAME, 'neuroglancer-layer-group-viewer')) + ) + screenshot = canvas_element.screenshot_as_png + with open('no_gain_screenshot.png', 'wb') as file: + file.write(screenshot) + print("Screenshot taken") + # Convert the screenshot to a NumPy array + image = Image.open(io.BytesIO(screenshot)) + no_gain_screenshot = np.array(image) + assert no_gain_screenshot.size != 0, "Image is empty" + # Check if the image contains valid pixel values + assert np.all(no_gain_screenshot >= 0) and np.all(no_gain_screenshot <= 255), "Image contains invalid pixel values" + no_gain_avg = np.mean(no_gain_screenshot) + print('No Gain average pixel value:') + print(no_gain_avg) + + + + + +@pytest.mark.timeout(10) +def test_gain_difference(): + assert gain_avg > no_gain_avg, "The gain screenshot is not brighter than the no gain screenshot" + + \ No newline at end of file diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 46d0dcc27..a38bd43e2 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -153,14 +153,18 @@ export function perspectivePanelEmitOIT(builder: ShaderBuilder) { } export function maxProjectionEmit(builder: ShaderBuilder) { - builder.addOutputBuffer("vec4", "v4f_fragData0", 0); - builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1); - builder.addOutputBuffer("highp vec4", "v4f_fragData2", 2); + builder.addOutputBuffer("vec4", "out_color", 0); + builder.addOutputBuffer("highp vec4", "out_z", 1); + builder.addOutputBuffer("highp vec4", "out_intensity", 2); + builder.addOutputBuffer("highp vec4", "out_pickId", 3); builder.addFragmentCode(` -void emit(vec4 color, float depth, float pick) { - v4f_fragData0 = color; - v4f_fragData1 = vec4(1.0 - depth, 1.0 - depth, 1.0 - depth, 1.0); - v4f_fragData2 = vec4(pick, pick, pick, 1.0); +void emit(vec4 color, float depth, float intensity, highp uint pickId) { + float pickIdFloat = float(pickId); + float bufferDepth = 1.0 - depth; + out_color = color; + out_z = vec4(bufferDepth, bufferDepth, bufferDepth, 1.0); + out_intensity = vec4(intensity, intensity, intensity, 1.0); + out_pickId = vec4(pickIdFloat, pickIdFloat, pickIdFloat, 1.0); }`); } @@ -168,6 +172,7 @@ const tempVec3 = vec3.create(); const tempVec4 = vec4.create(); const tempMat4 = mat4.create(); +// Copy the OIT values to the main color buffer function defineTransparencyCopyShader(builder: ShaderBuilder) { builder.addOutputBuffer("vec4", "v4f_fragColor", null); builder.setFragmentMain(` @@ -180,31 +185,45 @@ v4f_fragColor = vec4(accum.rgb / accum.a, revealage); `); } +// Copy the max projection color to the OIT buffer function defineMaxProjectionColorCopyShader(builder: ShaderBuilder) { - builder.addOutputBuffer("vec4", "v4f_fragColor", null); + builder.addOutputBuffer("vec4", "v4f_fragData0", 0); + builder.addOutputBuffer("vec4", "v4f_fragData1", 1); + builder.addFragmentCode(glsl_perspectivePanelEmitOIT); builder.setFragmentMain(` -v4f_fragColor = getValue0(); +vec4 color = getValue0(); +float bufferDepth = getValue1().r; +float weight = computeOITWeight(color.a, 1.0 - bufferDepth); +vec4 accum = color * weight; +float revealage = color.a; + +emitAccumAndRevealage(accum, revealage, 0u); `); } +// Copy the max projection depth and pick values to the main buffer function defineMaxProjectionPickCopyShader(builder: ShaderBuilder) { - builder.addOutputBuffer("vec4", "v4f_fragData0", 0); - builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1); - builder.addOutputBuffer("highp vec4", "v4f_fragData2", 2); + builder.addOutputBuffer("vec4", "out_color", 0); + builder.addOutputBuffer("highp vec4", "out_z", 1); + builder.addOutputBuffer("highp vec4", "out_pickId", 2); builder.setFragmentMain(` -v4f_fragData0 = vec4(0.0); -v4f_fragData1 = getValue0(); -v4f_fragData2 = getValue1(); +out_color = vec4(0.0); +out_z = getValue0(); +out_pickId = getValue1(); `); } +// Copy the max projection depth and picking to the max projection pick buffer. +// Note that the depth is set as the intensity value from the render layer. +// This is to combine max projection picking data via depth testing +// on the maximum intensity value of the data. function defineMaxProjectionToPickCopyShader(builder: ShaderBuilder) { - builder.addOutputBuffer("highp vec4", "v4f_fragData0", 0); - builder.addOutputBuffer("highp vec4", "v4f_fragData1", 1); + builder.addOutputBuffer("highp vec4", "out_z", 0); + builder.addOutputBuffer("highp vec4", "out_pickId", 1); builder.setFragmentMain(` -v4f_fragData0 = getValue0(); -v4f_fragData1 = getValue1(); -gl_FragDepth = v4f_fragData1.r; +out_z = getValue0(); +out_pickId = getValue2(); +gl_FragDepth = getValue1().r; `); } @@ -306,13 +325,13 @@ export class PerspectivePanel extends RenderedDataPanel { OffscreenCopyHelper.get(this.gl, defineTransparencyCopyShader, 2), ); protected maxProjectionColorCopyHelper = this.registerDisposer( - OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 1), + OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 2), ); protected maxProjectionPickCopyHelper = this.registerDisposer( OffscreenCopyHelper.get(this.gl, defineMaxProjectionPickCopyShader, 2), ); protected maxProjectionToPickCopyHelper = this.registerDisposer( - OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 2), + OffscreenCopyHelper.get(this.gl, defineMaxProjectionToPickCopyShader, 3), ); private sharedObject: PerspectiveViewState; @@ -730,6 +749,12 @@ export class PerspectivePanel extends RenderedDataPanel { WebGL2RenderingContext.RED, WebGL2RenderingContext.FLOAT, ), + new TextureBuffer( + this.gl, + WebGL2RenderingContext.R32F, + WebGL2RenderingContext.RED, + WebGL2RenderingContext.FLOAT, + ), ], depthBuffer: new DepthStencilRenderbuffer(this.gl), }), @@ -1004,6 +1029,7 @@ export class PerspectivePanel extends RenderedDataPanel { renderContext.depthBufferTexture = this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture; } + // Draw max projection layers if ( renderLayer.isVolumeRendering && isProjectionLayer(renderLayer as VolumeRenderingRenderLayer) @@ -1021,21 +1047,26 @@ export class PerspectivePanel extends RenderedDataPanel { bindMaxProjectionPickingBuffer(); this.maxProjectionToPickCopyHelper.draw( this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture, - this.maxProjectionConfiguration.colorBuffers[2 /*pick*/].texture, + this.maxProjectionConfiguration.colorBuffers[2 /*intensity*/] + .texture, + this.maxProjectionConfiguration.colorBuffers[3 /*pick*/].texture, ); - // Copy max projection color result to color only buffer + // Copy max projection color result to the transparent buffer with OIT // Depth testing off to combine max layers into one color via blend - this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR); + renderContext.bindFramebuffer(); gl.depthMask(false); gl.disable(WebGL2RenderingContext.DEPTH_TEST); gl.enable(WebGL2RenderingContext.BLEND); - gl.blendFunc( + gl.blendFuncSeparate( + WebGL2RenderingContext.ONE, WebGL2RenderingContext.ONE, + WebGL2RenderingContext.ZERO, WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA, ); this.maxProjectionColorCopyHelper.draw( this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture, + this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture, ); // Reset the max projection buffer @@ -1052,19 +1083,15 @@ export class PerspectivePanel extends RenderedDataPanel { gl.clearDepth(1.0); gl.clearColor(0.0, 0.0, 0.0, 1.0); gl.depthMask(false); - gl.blendFuncSeparate( - WebGL2RenderingContext.ONE, - WebGL2RenderingContext.ONE, - WebGL2RenderingContext.ZERO, - WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA, - ); gl.enable(WebGL2RenderingContext.DEPTH_TEST); gl.depthFunc(WebGL2RenderingContext.LESS); renderContext.emitter = perspectivePanelEmitOIT; renderContext.bindFramebuffer(); - continue; } - renderLayer.draw(renderContext, attachment); + // Draw regular transparent layers + else if (renderLayer.isTransparent) { + renderLayer.draw(renderContext, attachment); + } } // Copy transparent rendering result back to primary buffer. gl.disable(WebGL2RenderingContext.DEPTH_TEST); @@ -1109,27 +1136,21 @@ export class PerspectivePanel extends RenderedDataPanel { /*dppass=*/ WebGL2RenderingContext.REPLACE, ); gl.stencilMask(2); + if (hasMaxProjection) { + this.maxProjectionPickCopyHelper.draw( + this.maxProjectionPickConfiguration.colorBuffers[0].texture /*depth*/, + this.maxProjectionPickConfiguration.colorBuffers[1].texture /*pick*/, + ); + } for (const [renderLayer, attachment] of visibleLayers) { - if (!renderLayer.isTransparent || !renderLayer.transparentPickEnabled) { + if ( + !renderLayer.isTransparent || + !renderLayer.transparentPickEnabled || + renderLayer.isVolumeRendering + ) { + // Skip non-transparent layers and transparent layers with transparentPickEnabled=false. + // Volume rendering layers are handled separately and are combined in a pick buffer continue; - } - // For max projection layers, can copy over the pick buffer directly. - if (renderLayer.isVolumeRendering) { - if (isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)) { - this.maxProjectionPickCopyHelper.draw( - this.maxProjectionPickConfiguration.colorBuffers[0] - .texture /*depth*/, - this.maxProjectionPickConfiguration.colorBuffers[1] - .texture /*pick*/, - ); - } - // Draw picking for non min/max volume rendering layers - else { - // Currently volume rendering layers have no picking support - // Outside of min/max mode - continue; - } - // other transparent layers are drawn as usual } else { renderLayer.draw(renderContext, attachment); } diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 69d07d933..4361b1f2b 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -283,7 +283,7 @@ void emitRGBA(vec4 rgba) { savedIntensity = intensityChanged ? newIntensity : savedIntensity; savedDepth = intensityChanged ? depthAtRayPosition : savedDepth; outputColor = intensityChanged ? newColor : outputColor; - emit(outputColor, savedDepth, savedIntensity); + emit(outputColor, savedDepth, savedIntensity, uPickId); defaultMaxProjectionIntensity = 0.0; userEmittedIntensity = -100.0; `; @@ -311,6 +311,7 @@ void emitRGBA(vec4 rgba) { builder.addUniform("highp float", "uBrightnessFactor"); builder.addUniform("highp float", "uGain"); + builder.addUniform("highp uint", "uPickId"); builder.addVarying("highp vec4", "vNormalizedPosition"); builder.addTextureSampler( "sampler2D", @@ -368,7 +369,7 @@ vec2 computeUVFromClipSpace(vec4 clipSpacePosition) { `; if (isProjectionMode(shaderParametersState.mode)) { glsl_emitWireframe = ` - emit(outputColor, 1.0, uChunkNumber); + emit(outputColor, 1.0, uChunkNumber, uPickId); `; } builder.setFragmentMainFunction(` @@ -625,6 +626,9 @@ void main() { gl.enable(WebGL2RenderingContext.CULL_FACE); gl.cullFace(WebGL2RenderingContext.FRONT); + const pickId = isProjectionMode(this.mode.value) + ? renderContext.pickIDs.register(this) + : 0; forEachVisibleVolumeRenderingChunk( renderContext.projectionParameters, this.localPosition.value, @@ -802,6 +806,7 @@ void main() { } newSource = false; gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); + gl.uniform1ui(shader.uniform("uPickId"), pickId); drawBoxes(gl, 1, 1); ++presentCount; } else {