diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2249227d13..ddd40c5b20 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,15 +17,15 @@ jobs: matrix: include: - os: ubuntu-latest - java: 11 - - os: ubuntu-latest - java: 17 + java: 21 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - name: Setup Gradle + uses: gradle/gradle-build-action@v3 + - uses: gradle/wrapper-validation-action@v3 - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '${{ matrix.java }}' distribution: 'temurin' @@ -33,7 +33,9 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build scenery - run: ./gradlew build -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon + run: ./gradlew build testClasses -x test -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon + - name: Test scenery + run: ./gradlew test -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon windows-unittests: name: 'Windows' @@ -43,27 +45,29 @@ jobs: matrix: include: - os: windows-latest - java: 11 - - os: windows-latest - java: 17 + java: 21 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - name: Setup Gradle + uses: gradle/gradle-build-action@v3 + - uses: gradle/wrapper-validation-action@v3 - name: configure Pagefile - uses: al-cheb/configure-pagefile-action@v1.3 + uses: al-cheb/configure-pagefile-action@v1.4 with: minimum-size: 8GB maximum-size: 32GB disk-root: "D:" - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '${{ matrix.java }}' distribution: 'temurin' cache: 'gradle' - name: Build scenery - run: ./gradlew build -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon + run: ./gradlew build testClasses -x test -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon + - name: Test scenery + run: ./gradlew test -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon mac-unittests: name: 'macOS' @@ -73,15 +77,15 @@ jobs: matrix: include: - os: macos-latest - java: 11 - - os: macos-latest - java: 17 + java: 21 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - name: Setup Gradle + uses: gradle/gradle-build-action@v3 + - uses: gradle/wrapper-validation-action@v3 - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '${{ matrix.java }}' distribution: 'temurin' @@ -89,4 +93,6 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build scenery - run: ./gradlew build -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon + run: ./gradlew build testClasses -x test -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon + - name: Test scenery + run: ./gradlew test -x dokkaHtml -x dokkaHtmlJar -x javadoc -x dokkaJavadocJar --no-daemon diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a645adb5e0..d4a1764e23 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,23 +5,28 @@ cache: paths: - pdb-store/ - .gradle-user-home/ - - node_modules/ - - screenshots/ + +workflow: + rules: + - if: $CI_COMMIT_MESSAGE =~ /-draft$/ + when: never + - if: $CI_COMMIT_MESSAGE =~ /-WIP$/ + when: never + - if: $CI_PIPELINE_SOURCE == "push" variables: - JAVA_HOME: "/usr/lib/jvm/java-11-openjdk-amd64" + JAVA_HOME: "/usr/lib/jvm/java-21-openjdk-amd64" PDB_CACHE_DIR: "./pdb-store/cache" PDB_DIR: "./pdb-store" MODEL_DIR: "./models" GRADLE_USER_HOME: "./.gradle-user-home" + VALIDATE_VULKAN: "false" # simple unit tests without requiring GPUs unit-tests-no-gpu: - image: ubuntu:bionic + image: scenerygraphics/nvidia-vulkan:1.3.261.1-ubuntu20.04-v3 before_script: - mkdir -p ./pdb-store/cache - - apt-get update -qq --force-yes > /dev/null - - apt-get install -qq -y --allow-downgrades --allow-remove-essential --allow-change-held-packages unzip wget git curl libgl1-mesa-glx sudo sed openjdk-11-jdk-headless > /dev/null - if [ ! -d "$MODEL_DIR" ]; then wget -q https://ulrik.is/scenery-demo-models.zip && unzip -q scenery-demo-models.zip; fi - chmod +x gradlew script: @@ -30,15 +35,17 @@ unit-tests-no-gpu: # base job for running with GPUs .base-job-gpu: &base-job-gpu before_script: + # $VULKAN_SDK_VERSION comes from Docker image + - source /opt/vulkan/$VULKAN_SDK_VERSION/setup-env.sh - mkdir -p ./pdb-store/cache - # Installs Maven, Vulkan development libraries, etc. - - apt-get update -qq --force-yes > /dev/null - - apt-get install -qq -y curl sudo sed > /dev/null - sudo sed -i -e '/^assistive_technologies=/s/^/#/' /etc/java-*-openjdk/accessibility.properties # Output Vulkan driver information, but do not fail in case of non-zero # return (happens e.g. if $DISPLAY is not set) + - echo -e "\e[0Ksection_start:`date +%s`:hw_info_section[collapsed=true]\r\e[0KHardware Information" + - nvidia-smi || true - vulkaninfo || true - clinfo || true + - echo -e "\e[0Ksection_end:`date +%s`:hw_info_section\r\e[0K" - if [ ! -d "$MODEL_DIR" ]; then wget -q https://ulrik.is/scenery-demo-models.zip && unzip -q scenery-demo-models.zip; fi - chmod +x gradlew - ./gradlew --stop # stop any deamon https://stackoverflow.com/a/58397542/1047713 @@ -48,28 +55,28 @@ unit-tests-no-gpu: - echo -e "\e[0Ksection_end:`date +%s`:build_section\r\e[0K" # basic group - echo -e "\e[0Ksection_start:`date +%s`:basic_section[collapsed=true]\r\e[0KBasic Test Group" - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=basic -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 -Dscenery.ExampleRunner.Blocklist=EdgeBundlerExample - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=basic -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 -Dscenery.ExampleRunner.Blocklist=EdgeBundlerExample + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=basic -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 -Dscenery.ExampleRunner.Blocklist=EdgeBundlerExample -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=basic -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 -Dscenery.ExampleRunner.Blocklist=EdgeBundlerExample -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 - echo -e "\e[0Ksection_end:`date +%s`:basic_section\r\e[0K" # advanced group - echo -e "\e[0Ksection_start:`date +%s`:advanced_section[collapsed=true]\r\e[0KAdvanced Test Group" - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=advanced -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=advanced -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=advanced -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=advanced -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 - echo -e "\e[0Ksection_end:`date +%s`:advanced_section\r\e[0K" # compute group - echo -e "\e[0Ksection_start:`date +%s`:compute_section[collapsed=true]\r\e[0KCompute Test Group" - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=compute -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=compute -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=compute -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=compute -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 - echo -e "\e[0Ksection_end:`date +%s`:compute_section\r\e[0K" # volumes group - echo -e "\e[0Ksection_start:`date +%s`:volumes_section[collapsed=true]\r\e[0KVolumes Test Group" - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=volumes -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 - - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=volumes -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=volumes -Dscenery.ExampleRunner.Configurations=DeferredShading.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 + - ./gradlew test jacocoTestReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=volumes -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 - echo -e "\e[0Ksection_end:`date +%s`:volumes_section\r\e[0K" # code coverage reporting - echo -e "\e[0Ksection_start:`date +%s`:coverage_section[collapsed=true]\r\e[0KCode Coverage and Analysis" # we keep the same arguments here as in the last test run to not startle Gradle into re-running the test task - - ./gradlew fullCodeCoverageReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=volumes -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 + - ./gradlew fullCodeCoverageReport --build-cache --full-stacktrace -Pgpu=true -Dscenery.ExampleRunner.TestGroup=volumes -Dscenery.ExampleRunner.Configurations=DeferredShadingStereo.yml -Dscenery.RandomSeed=13371842 -Dscenery.VulkanRenderer.EnableValidations=$VALIDATE_VULKAN -Dscenery.VulkanRenderer.StrictValidation=true -Dscenery.VulkanRenderer.DefaultRenderDelay=100 - echo -e "\e[0Ksection_end:`date +%s`:coverage_section\r\e[0K" artifacts: when: always @@ -79,46 +86,12 @@ unit-tests-no-gpu: - "hs_err_*" scenery-nvidia: - image: scenerygraphics/nvidia-vulkan:1.3.216.0-ubuntu20.04-updatedmodels + image: scenerygraphics/nvidia-vulkan:1.3.261.1-ubuntu20.04-v3 <<: *base-job-gpu after_script: - - nvidia-smi - tar cvjf results.tar.bz2 screenshots/ - - npm install @argos-ci/cli - - ARGOS_COMMIT=$CI_COMMIT_SHA ARGOS_BRANCH=$CI_COMMIT_REF_NAME ./node_modules/\@argos-ci/cli/bin/argos-cli.js upload screenshots/ || true + - ARGOS_COMMIT=$CI_COMMIT_SHA ARGOS_BRANCH=$CI_COMMIT_REF_NAME argos upload screenshots/ || true tags: - cuda - intel -#scenery-amd: -# image: rocm/rocm-terminal -# <<: *base-job -# variables: -# SUDO: "sudo" -# GPURUN: "sudo su -m - rocm-user -c" -# before_script: -# # The rocm docker container requires the user to be in the video group which -# # can usually be set via docker's --group-add option. GitLab-Runner currently -# # has no known option for doing that. Therefore, it manually has to happen in -# # the job description. -# - $SUDO usermod -a -G video rocm-user -# # Installs Maven, Vulkan development libraries, etc. -# - $SUDO apt-get -qq --force-yes update > /dev/null -# - $SUDO apt-get install -qq --force-yes unzip kmod wget git maven openjdk-8-jdk libvulkan1 libvulkan-dev vulkan-utils > /dev/null -# # Installs the AMD GPUopen Vulkan driver -# - wget https://github.com/GPUOpen-Drivers/AMDVLK/releases/download/v-2019.Q3.6/amdvlk_2019.Q3.6_amd64.deb -# - $SUDO dpkg -i amdvlk_2019.Q3.6_amd64.deb -# - $SUDO apt-get -f install -# - $SUDO lsmod -# # Output Vulkan driver information, but do not fail in case of non-zero -# # return (happens e.g. if $DISPLAY is not set) -# - vulkaninfo || true -# - wget -q https://ulrik.is/scenery-demo-models.zip -# - unzip -q scenery-demo-models.zip -# after_script: -# - rocm-smi -# tags: -# - amd -# - rocm -# -# diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md new file mode 100644 index 0000000000..6e402573b9 --- /dev/null +++ b/ACKNOWLEDGEMENTS.md @@ -0,0 +1,48 @@ +# Acknowledgements + +## Colormaps +scenery uses colormaps from the NCL project, licensed under the following terms: + +> Copyright © 2007-2018 University Corporation for Atmospheric Research (UCAR). All rights reserved. Developed by NCAR's Computational and Information Systems Laboratory, UCAR, www.cisl.ucar.edu, with the following contributions: +> * dcdflib The University of Texas, M.D. Anderson Cancer Center http://www.netlib.org/random/ +> * GRIBEX Copyright © 1981-2007 +> European Centre for Medium-Range Weather Forecast http://www.ecmwf.int/products/data/software/grib.html +> * LAPACK Copyright © 1999 +> Society for Industrial and Applied Mathematics http://www.netlib.org/lapack/lug/lapack_lug.html +> * random The University of Texas, M.D. Anderson Cancer Center http://www.netlib.org/random/ +> * RANGS / GSHHS Dr. Rainer Feistel +> Baltic Sea Research Institute Warnemunde https://www.io-warnemuende.de/rangs-en.html +> * Spherepack Copyright © December 2011 +> * UCAR http://www2.cisl.ucar.edu/resources/legacy/spherepack/ +> * UDUNITS-2 Copyright © 2012 University Corporation for Atmospheric Research +> UCAR/Unidata http://www.unidata.ucar.edu/software/udunits/udunits-2/udunits2.html +> * Udunits extensions from ncview David Pierce, UCSD +> http://meteora.ucsd.edu/~pierce/ncview_home_page.html +> Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +> +> Neither the names of NCAR's Computational and Information Systems Laboratory, the University Corporation for Atmospheric Research, nor the names of its contributors may be used to endorse or promote products derived from this Software without specific prior written permission. +> Redistributions of source code must retain the above copyright notices, this list of conditions, and the disclaimer below. +> Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and the disclaimer below in the documentation and/or other materials provided with the distribution. +> THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE SOFTWARE. +> + +## FXAA Implementation + +scenery uses an implementation of FXAA based on [Nvidia's original paper](http://developer.download.nvidia.com/assets/gamedev/files/sdk/11/FXAA_WhitePaper.pdf) and [McNopper's FXAA implementation](https://github.com/McNopper/OpenGL/blob/master/Example42/shader/fxaa.frag.glsl). + +## Icons + +scenery uses icons from the [FontAwesome](https://fontawesome.com) project, licensed under the CC-Attribution 4.0 license: +> Font Awesome Free License +> +> Font Awesome Free is free, open source, and GPL friendly. You can use it for +> commercial projects, open source projects, or really almost whatever you want. +> Full Font Awesome Free license: https://fontawesome.com/license/free. +> +> -------------------------------------------------------------------------------- +> +> Icons: CC BY 4.0 License (https://creativecommons.org/licenses/by/4.0/) +> +> The Font Awesome Free download is licensed under a Creative Commons +> Attribution 4.0 International License and applies to all icons packaged +> as SVG and JS file types. diff --git a/README.md b/README.md index d2d021d344..e06e7f460f 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ [![Join the scenery+sciview Zulip chat](https://img.shields.io/badge/zulip-join_chat-brightgreen.svg)](https://imagesc.zulipchat.com/#narrow/stream/327495-scenery.2Bsciview) -[![pipeline status](https://gitlab.com/hzdr/crp/scenery/badges/master/pipeline.svg)](https://gitlab.com/hzdr/crp/scenery/-/commits/master) -[![Build Status](https://github.com/scenerygraphics/scenery/workflows/build/badge.svg)](https://github.com/scenerygraphics/scenery/actions?workflow=build) +[![Gitlab CI Status](https://gitlab.com/hzdr/crp/scenery/badges/master/pipeline.svg)](https://gitlab.com/hzdr/crp/scenery/-/commits/master) +[![Github Actions Status](https://github.com/scenerygraphics/scenery/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/scenerygraphics/scenery/actions/workflows/build.yml) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/2178c074ae254f3694529bff94528747)](https://www.codacy.com/gh/scenerygraphics/scenery/dashboard?utm_source=github.com&utm_medium=referral&utm_content=scenerygraphics/scenery&utm_campaign=Badge_Grade) # scenery // Flexible VR Visualisation for Volumetric and Geometric Data on the Java VM @@ -98,14 +98,14 @@ The recommended way to use non-release (unstable) builds is to use jitpack. jitp ### Using _scenery_ in a Maven project -Add scenery and ClearGL to your project's `pom.xml`: +Add scenery to your project's `pom.xml`: ```xml graphics.scenery scenery - 0.9.0 + 0.11.0 ``` @@ -141,7 +141,7 @@ Add scenery to your project's `build.gradle`: ```kotlin dependencies { // ... - api("graphics.scenery:scenery:0.9.0") + api("graphics.scenery:scenery:0.11.0") } ``` diff --git a/build-example-project.sh b/build-example-project.sh new file mode 100755 index 0000000000..d73eb848e2 --- /dev/null +++ b/build-example-project.sh @@ -0,0 +1,23 @@ +#!/bin/bash +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +set -e +originalVersion=`cat gradle.properties | grep version | cut -d "=" -f2` +gitHash=`git rev-parse HEAD` +version="$originalVersion-${gitHash:0:7}" + +echo -e "${RED}Original scenery version is $originalVersion, modified version for test build is $version${NC}" + +echo -e "*** ${GREEN}Installing $version to local Maven repository...${NC}" +./gradlew publishToMavenLocal -Pversion=$version + +echo -e "*** ${GREEN}Cloning test project ...${NC}" +rm -rf build/testProject +mkdir -p build/testProject +cd build/testProject +git clone https://github.com/scenerygraphics/minimal-scenery-example-project . + +echo -e "*** ${GREEN}Building test project against scenery $version ...${NC}" +./gradlew --no-build-cache build -PsceneryVersion=$version -PmavenLocal=true diff --git a/build.gradle.kts b/build.gradle.kts index 943b7a47bd..0509d5c23d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -35,7 +35,10 @@ val lwjglArtifacts = listOf( "lwjgl-remotery", "lwjgl-spvc", "lwjgl-shaderc", - "lwjgl-tinyexr" + "lwjgl-tinyexr", + "lwjgl-jawt", + "lwjgl-lz4", + "lwjgl-zstd" ) dependencies { @@ -43,20 +46,20 @@ dependencies { val lwjglVersion = project.properties["lwjglVersion"] implementation(platform("org.scijava:pom-scijava:$scijavaParentPomVersion")) - annotationProcessor("org.scijava:scijava-common:2.96.0") + annotationProcessor("org.scijava:scijava-common:2.98.0") implementation(kotlin("reflect")) - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0") implementation("org.slf4j:slf4j-api:1.7.36") implementation("org.joml:joml:1.10.5") - implementation("net.java.jinput:jinput:2.0.9", "natives-all") + implementation("net.java.jinput:jinput:2.0.10", "natives-all") implementation("org.jocl:jocl:2.0.5") implementation("org.scijava:scijava-common") implementation("org.scijava:script-editor") implementation("org.scijava:ui-behaviour") implementation("org.scijava:scripting-jython") - implementation("net.java.dev.jna:jna-platform:5.11.0") + implementation("net.java.dev.jna:jna-platform:5.14.0") lwjglArtifacts.forEach { artifact -> @@ -81,6 +84,9 @@ dependencies { } } + // JAWT doesn't bring natives along + artifact.endsWith("jawt") -> {} + else -> { logger.info("else: org.lwjgl:$artifact:$lwjglVersion:$native") runtimeOnly("org.lwjgl:$artifact:$lwjglVersion:$native") @@ -88,34 +94,36 @@ dependencies { } } } - implementation("com.fasterxml.jackson.core:jackson-databind:2.13.3") - implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.3") - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.3") - implementation("org.zeromq:jeromq:0.5.3") - implementation("com.esotericsoftware:kryo:5.5.0") + implementation("org.xerial.snappy:snappy-java:1.1.10.5") + implementation("com.fasterxml.jackson.core:jackson-databind:2.17.0") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.17.0") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.0") + implementation("org.zeromq:jeromq:0.6.0") + implementation("com.esotericsoftware:kryo:5.6.0") implementation("de.javakaffee:kryo-serializers:0.45") - implementation("org.msgpack:msgpack-core:0.9.5") - implementation("org.msgpack:jackson-dataformat-msgpack:0.9.1") + implementation("org.msgpack:msgpack-core:0.9.8") + implementation("org.msgpack:jackson-dataformat-msgpack:0.9.8") api("graphics.scenery:jvrpn:1.2.0", lwjglNatives.filter { !it.contains("arm") }.toTypedArray()) implementation("io.scif:scifio") - implementation("org.bytedeco:ffmpeg:6.0-1.5.9", ffmpegNatives) - implementation("io.github.classgraph:classgraph:4.8.161") + implementation("org.bytedeco:ffmpeg:6.1.1-1.5.10", ffmpegNatives) + implementation("io.github.classgraph:classgraph:4.8.170") - implementation("info.picocli:picocli:4.7.4") + implementation("info.picocli:picocli:4.7.5") - api("sc.fiji:bigdataviewer-core:10.4.10") + api("sc.fiji:bigdataviewer-core:10.4.14") api("sc.fiji:bigdataviewer-vistools:1.0.0-beta-28") - api("sc.fiji:bigvolumeviewer:0.3.1") { + api("sc.fiji:bigvolumeviewer:0.3.3") { exclude("org.jogamp.gluegen", "gluegen-rt") exclude("org.jogamp.jogl", "jogl-all") } - implementation("com.github.lwjglx:lwjgl3-awt:bc8daf5") { + implementation("com.github.skalarproduktraum:lwjgl3-awt:c034a77") { // we exclude the LWJGL binaries here, as the lwjgl3-awt POM uses // Maven properties for natives, which is not supported by Gradle exclude("org.lwjgl", "lwjgl-bom") exclude("org.lwjgl", "lwjgl") } + implementation("org.janelia.saalfeldlab:n5") implementation("org.janelia.saalfeldlab:n5-imglib2") listOf("core", "structure", "modfinder").forEach { @@ -126,7 +134,7 @@ dependencies { exclude("org.biojava.thirdparty", "forester") } } - implementation("org.jetbrains.kotlin:kotlin-scripting-jsr223:1.5.31") + implementation("org.jetbrains.kotlin:kotlin-scripting-jsr223:1.9.23") api("graphics.scenery:art-dtrack-sdk:2.6.0") testImplementation(kotlin("test")) @@ -139,7 +147,7 @@ dependencies { testImplementation("net.imagej:ij") testImplementation("net.imglib2:imglib2-ij") - implementation("org.jfree:jfreechart:1.5.0") + implementation("org.jfree:jfreechart:1.5.4") implementation("net.imagej:imagej-ops:0.45.5") } @@ -149,14 +157,14 @@ val isRelease: Boolean tasks { withType().all { kotlinOptions { - jvmTarget = project.properties["jvmTarget"]?.toString() ?: "11" + jvmTarget = project.properties["jvmTarget"]?.toString() ?: "21" freeCompilerArgs += listOf("-Xinline-classes", "-opt-in=kotlin.RequiresOptIn") } } withType().all { - targetCompatibility = project.properties["jvmTarget"]?.toString() ?: "11" - sourceCompatibility = project.properties["jvmTarget"]?.toString() ?: "11" + targetCompatibility = project.properties["jvmTarget"]?.toString() ?: "21" + sourceCompatibility = project.properties["jvmTarget"]?.toString() ?: "21" } withType().configureEach { @@ -226,6 +234,11 @@ tasks { return@pkg } + // JAWT doesn't have any natives + if(lwjglProject.contains("jawt")) { + return@pkg + } + if(lwjglProject.contains("openvr") && nativePlatform.contains("mac") && nativePlatform.contains("arm64")) { @@ -289,7 +302,8 @@ tasks { "jackson-module-kotlin", "jackson-dataformat-yaml", "kryo", - "bigvolumeviewer" + "bigvolumeviewer", + "snappy-java" ) + lwjglArtifacts val toSkip = listOf("pom-scijava") @@ -372,9 +386,9 @@ tasks { enabled = isRelease dokkaSourceSets.configureEach { sourceLink { - localDirectory.set(file("src/main/kotlin")) - remoteUrl.set(URL("https://github.com/scenerygraphics/scenery/tree/main/src/main/kotlin")) - remoteLineSuffix.set("#L") + localDirectory = file("src/main/kotlin") + remoteUrl = URL("https://github.com/scenerygraphics/scenery/tree/main/src/main/kotlin") + remoteLineSuffix = "#L" } } } @@ -391,7 +405,7 @@ tasks { } } -jacoco.toolVersion = "0.8.8" +jacoco.toolVersion = "0.8.11" java.withSourcesJar() @@ -399,10 +413,7 @@ plugins.withType { tasks.test { finalizedBy("jacocoTestReport") } } -// disable Gradle metadata file creation on Jitpack, as jitpack modifies -// the metadata file, resulting in broken metadata with missing native dependencies. -if(System.getenv("JITPACK") != null) { - tasks.withType { - enabled = false - } +// disable Gradle metadata file in general, as Maven artifacts are our main publication. +tasks.withType { + enabled = false } diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index ea6b71c717..d3337eac24 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -10,7 +10,7 @@ repositories { } dependencies { - implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.8.20") + implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.9.20") } //, imagej, imgLib2, scifio, fiji, bigDataViewer, trakEM2, n5, boneJ, ome, omero, diff --git a/buildSrc/src/main/kotlin/scenery/base.gradle.kts b/buildSrc/src/main/kotlin/scenery/base.gradle.kts index 977c080f90..aed635f792 100644 --- a/buildSrc/src/main/kotlin/scenery/base.gradle.kts +++ b/buildSrc/src/main/kotlin/scenery/base.gradle.kts @@ -7,6 +7,11 @@ plugins { jacoco } +repositories { + mavenCentral() +} + +jacoco.toolVersion = "0.8.11" tasks { // https://docs.gradle.org/current/userguide/java_testing.html#test_filtering @@ -50,7 +55,7 @@ tasks { val testConfig = System.getProperty("scenery.ExampleRunner.Configurations", "None") configure { - setDestinationFile(file("$buildDir/jacoco/jacocoTest.$testGroup.$testConfig.exec")) + setDestinationFile(file("${layout.buildDirectory}/jacoco/jacocoTest.$testGroup.$testConfig.exec")) println("Destination file for jacoco is $destinationFile (test, $testGroup, $testConfig)") } @@ -59,7 +64,7 @@ tasks { // this should circumvent Nvidia's Vulkan cleanup issue maxParallelForks = 1 setForkEvery(1) - systemProperty("scenery.Workarounds.DontCloseVulkanInstances", "true") + //systemProperty("scenery.Workarounds.DontCloseVulkanInstances", "true") testLogging { exceptionFormat = TestExceptionFormat.FULL @@ -77,12 +82,8 @@ tasks { val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } println("Adding properties ${props.size}/$props") - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } + allJvmArgs = allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + System.getenv("SCENERY_JVM_ARGS")?.let { allJvmArgs = allJvmArgs + it } } finalizedBy(jacocoTestReport) // report is always generated after tests run @@ -101,7 +102,7 @@ tasks { register("compileShader") { group = "tools" - mainClass.set("graphics.scenery.backends.ShaderCompiler") + mainClass = "graphics.scenery.backends.ShaderCompiler" classpath = sourceSets["main"].runtimeClasspath } @@ -111,19 +112,19 @@ tasks { sourceSets(sourceSets["main"], sourceSets["test"]) reports { - xml.required.set(true) - html.required.set(true) - csv.required.set(false) + xml.required = true + html.required = true + csv.required = false } dependsOn(test) } named("jar") { - archiveVersion.set(rootProject.version.toString()) + archiveVersion = rootProject.version.toString() manifest.attributes["Implementation-Build"] = run { // retrieve the git commit hash val gitFolder = "$projectDir/.git/" - val digit = 6 + val digit = 7 /* '.git/HEAD' contains either * in case of detached head: the currently checked out commit hash * otherwise: a reference to a file containing the current commit hash */ @@ -136,12 +137,14 @@ tasks { .readText() }.trim().take(digit) } + + manifest.attributes["Implementation-Version"] = project.version } jacocoTestReport { reports { - xml.required.set(true) - html.required.set(false) + xml.required = true + html.required = false } dependsOn(test) // tests are required to run before generating the report } @@ -156,17 +159,13 @@ tasks { register(name = exampleName) { classpath = sourceSets.test.get().runtimeClasspath - mainClass.set(className) + mainClass = className group = "examples.$exampleType" val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } + allJvmArgs = allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + System.getenv("SCENERY_JVM_ARGS")?.let { allJvmArgs = allJvmArgs + it } if(JavaVersion.current() > JavaVersion.VERSION_11) { allJvmArgs = allJvmArgs + listOf( @@ -191,15 +190,11 @@ tasks { if (project.hasProperty("example")) { project.property("example")?.let { example -> val file = sourceSets.test.get().allSource.files.first { "class $example" in it.readText() } - mainClass.set(file.path.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt")) + mainClass = file.path.substringAfter("kotlin${File.separatorChar}").replace(File.separatorChar, '.').substringBefore(".kt") val props = System.getProperties().filter { (k, _) -> k.toString().startsWith("scenery.") } - val additionalArgs = System.getenv("SCENERY_JVM_ARGS") - allJvmArgs = if (additionalArgs != null) { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + additionalArgs - } else { - allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } - } + allJvmArgs = allJvmArgs + props.flatMap { (k, v) -> listOf("-D$k=$v") } + System.getenv("SCENERY_JVM_ARGS")?.let { allJvmArgs = allJvmArgs + it } if(JavaVersion.current() > JavaVersion.VERSION_11) { allJvmArgs = allJvmArgs + listOf( @@ -222,6 +217,122 @@ tasks { } } } + + // this is the Kotlinized version of https://github.com/mendhak/Gradle-Github-Colored-Output + // (c) by Mendhak, with a few adjustments, e.g. to cache failures and skips, and print them + // in the summary. + withType(Test::class.java) { + val folding = System.getenv("GITHUB_ACTIONS") != null + + val ANSI_BOLD_WHITE = "\u001B[01m" + val ANSI_RESET = "\u001B[0m" + val ANSI_BLACK = "\u001B[30m" + val ANSI_RED = "\u001B[31m" + val ANSI_GREEN = "\u001B[32m" + val ANSI_YELLOW = "\u001B[33m" + val ANSI_BLUE = "\u001B[34m" + val ANSI_PURPLE = "\u001B[35m" + val ANSI_CYAN = "\u001B[36m" + val ANSI_WHITE = "\u001B[37m" + val CHECK_MARK = "\u2713" + val NEUTRAL_FACE = "\u0CA0_\u0CA0" + val X_MARK = "\u274C" + + addTestListener(object: TestListener { + val failures = ArrayList() + val skips = ArrayList() + + override fun beforeSuite(suite: TestDescriptor?) { + if(suite == null) { + return + } + + if(suite.name.startsWith("Test Run") || suite.name.startsWith("Gradle Worker")) return + + if(suite.parent != null && suite.className != null) { + if(folding) { + println("##[group]" + suite.name + "\r") + } + println(ANSI_BOLD_WHITE + suite.name + ANSI_RESET) + } + + } + + override fun afterTest(descriptor: TestDescriptor?, result: TestResult) { + var indicator = ANSI_WHITE + + indicator = if(result.failedTestCount > 0) ANSI_RED + X_MARK + else if(result.skippedTestCount > 0) ANSI_YELLOW + NEUTRAL_FACE + else ANSI_GREEN + CHECK_MARK + + println(" " + indicator + ANSI_RESET + " " + descriptor?.name) + + if(result.failedTestCount > 0) { + println(" ") + failures.add("${descriptor?.parent}:${descriptor?.name}") + } + if(result.skippedTestCount > 0) { + skips.add("${descriptor?.parent}:${descriptor?.name}") + } + + } + + override fun afterSuite(desc: TestDescriptor?, result: TestResult) { + if(desc == null) { + return + } + + if(desc.parent != null && desc.className != null) { + if(folding && result.failedTestCount == 0L) { + println("##[endgroup]\r") + println("") + } + } + + // will match the outermost suite + if(desc.parent == null) { + var failStyle = ANSI_RED + var skipStyle = ANSI_YELLOW + + if(result.failedTestCount > 0) { + failStyle = ANSI_RED + } + + if(result.skippedTestCount > 0) { + skipStyle = ANSI_YELLOW + } + + val summaryStyle = when(result.resultType) { + TestResult.ResultType.SUCCESS -> ANSI_GREEN + TestResult.ResultType.FAILURE -> ANSI_RED + else -> ANSI_WHITE + } + + println("--------------------------------------------------------------------------") + println( + "Results: " + summaryStyle + "${result.resultType}" + ANSI_RESET + + " (${result.testCount} tests, " + + ANSI_GREEN + "${result.successfulTestCount} passed" + ANSI_RESET + + ", " + failStyle + "${result.failedTestCount} failed" + ANSI_RESET + + ", " + skipStyle + "${result.skippedTestCount} skipped" + ANSI_RESET + + ")" + ) + + if(result.failedTestCount > 0) { + println(failStyle + "Failed tests:\n${failures.joinToString("\n") { " * $it "}}" + ANSI_RESET) + } + + if(result.skippedTestCount> 0) { + println(skipStyle + "Skipped tests:\n${skips.joinToString("\n") { " * $it "}}" + ANSI_RESET) + } + println("--------------------------------------------------------------------------") + } + } + + override fun beforeTest(testDescriptor: TestDescriptor?) { + } + }) + } } val TaskContainer.jacocoTestReport: TaskProvider diff --git a/buildSrc/src/main/kotlin/scenery/publish.gradle.kts b/buildSrc/src/main/kotlin/scenery/publish.gradle.kts index 162487eda1..284857621b 100644 --- a/buildSrc/src/main/kotlin/scenery/publish.gradle.kts +++ b/buildSrc/src/main/kotlin/scenery/publish.gradle.kts @@ -16,18 +16,18 @@ tasks { dokkaHtml { dokkaSourceSets.configureEach { sourceLink { - localDirectory.set(file("src/main/kotlin")) - remoteUrl.set(URL("https://github.com/scenerygraphics/scenery/tree/main/src/main/kotlin")) - remoteLineSuffix.set("#L") + localDirectory = file("src/main/kotlin") + remoteUrl = URL("https://github.com/scenerygraphics/scenery/tree/main/src/main/kotlin") + remoteLineSuffix = "#L" } } } dokkaJavadoc { dokkaSourceSets.configureEach { sourceLink { - localDirectory.set(file("src/main/kotlin")) - remoteUrl.set(URL("https://github.com/scenerygraphics/scenery/tree/main/src/main/kotlin")) - remoteLineSuffix.set("#L") + localDirectory = file("src/main/kotlin") + remoteUrl = URL("https://github.com/scenerygraphics/scenery/tree/main/src/main/kotlin") + remoteLineSuffix = "#L" } } } @@ -44,13 +44,13 @@ publishing { val dokkaJavadocJar by tasks.register("dokkaJavadocJar") { dependsOn(tasks.dokkaJavadoc) from(tasks.dokkaJavadoc.flatMap { it.outputDirectory }) - archiveClassifier.set("javadoc") + archiveClassifier = "javadoc" } val dokkaHtmlJar by tasks.register("dokkaHtmlJar") { dependsOn(tasks.dokkaHtml) from(tasks.dokkaHtml.flatMap { it.outputDirectory }) - archiveClassifier.set("html-doc") + archiveClassifier = "html-doc" } @@ -59,72 +59,72 @@ publishing { // TODO, resolved dependencies versions? https://docs.gradle.org/current/userguide/publishing_maven.html#publishing_maven:resolved_dependencies pom { - name.set(rootProject.name) - description.set(rootProject.description) - url.set(sceneryUrl) - properties.set(mapOf("inceptionYear" to "2016")) + name = rootProject.name + description = rootProject.description + url = sceneryUrl + inceptionYear = "2016" organization { - name.set(rootProject.name) - url.set(sceneryUrl) + name = rootProject.name + url = sceneryUrl } licenses { license { - name.set("GNU Lesser General Public License v3+") - url.set("https://www.gnu.org/licenses/lgpl.html") - distribution.set("repo") + name = "GNU Lesser General Public License v3+" + url = "https://www.gnu.org/licenses/lgpl.html" + distribution = "repo" } } developers { developer { - id.set("skalarproduktraum") - name.set("Ulrik Guenther") - url.set("https://ulrik.is/writing") + id = "skalarproduktraum" + name = "Ulrik Guenther" + url = "https://ulrik.is/writing" roles.addAll("founder", "lead", "developer", "debugger", "reviewer", "support", "maintainer") } } contributors { contributor { - name.set("Kyle Harrington") - url.set("http://www.kyleharrington.com") - properties.set(mapOf("id" to "kephale")) + name = "Kyle Harrington" + url = "http://www.kyleharrington.com" + properties = mapOf("id" to "kephale") } contributor { - name.set("Tobias Pietzsch") - url.set("https://imagej.net/people/tpietzsch") - properties.set(mapOf("id" to "tpietzsch")) + name = "Tobias Pietzsch" + url = "https://imagej.net/people/tpietzsch" + properties = mapOf("id" to "tpietzsch") } contributor { - name.set("Loic Royer") - properties.set(mapOf("id" to "royerloic")) + name = "Loic Royer" + properties = mapOf("id" to "royerloic") } contributor { - name.set("Martin Weigert") - properties.set(mapOf("id" to "maweigert")) + name = "Martin Weigert" + properties = mapOf("id" to "maweigert") } contributor { - name.set("Aryaman Gupta") - properties.set(mapOf("id" to "aryaman-gupta")) + name = "Aryaman Gupta" + properties = mapOf("id" to "aryaman-gupta") } contributor { - name.set("Curtis Rueden") - url.set("https://imagej.net/people/ctrueden") - properties.set(mapOf("id" to "ctrueden")) + name = "Curtis Rueden" + url = "https://imagej.net/people/ctrueden" + properties = mapOf("id" to "ctrueden") } } - mailingLists { mailingList { name.set("none") } } + mailingLists { mailingList { name = "none" } } scm { - connection.set("scm:git:https://github.com/scenerygraphics/scenery") - developerConnection.set("scm:git:git@github.com:scenerygraphics/scenery") - tag.set(if(snapshot) "HEAD" else "scenery-${rootProject.version}") - url.set(sceneryUrl) + connection = "scm:git:https://github.com/scenerygraphics/scenery" + developerConnection = "scm:git:git@github.com:scenerygraphics/scenery" + tag = if(snapshot) "HEAD" else "scenery-${rootProject.version}" + url = sceneryUrl } issueManagement { - system.set("GitHub Issues") - url.set("https://github.com/scenerygraphics/scenery/issues") + system = "GitHub Issues" + url = "https://github.com/scenerygraphics/scenery/issues" } ciManagement { - system.set("GitHub Actions") - url.set("https://github.com/scenerygraphics/scenery/actions") + system = "GitHub Actions" + url = "https://github.com/scenerygraphics/scenery/actions" } distributionManagement { // https://stackoverflow.com/a/21760035/1047713 diff --git a/buildSrc/src/main/kotlin/scenery/utils.kt b/buildSrc/src/main/kotlin/scenery/utils.kt index 6d671185ae..dd569a14a8 100644 --- a/buildSrc/src/main/kotlin/scenery/utils.kt +++ b/buildSrc/src/main/kotlin/scenery/utils.kt @@ -71,6 +71,5 @@ fun DependencyHandlerScope.api(dep: String, natives: Array) { native, null, null) } -val joglNatives = arrayOf("natives-windows-amd64", "natives-linux-i586", "natives-linux-amd64", "natives-macosx-universal") val lwjglNatives = arrayOf("natives-windows", "natives-linux", "natives-macos", "natives-macos-arm64") val ffmpegNatives = arrayOf("windows-x86_64", "linux-x86_64", "macosx-x86_64", "macosx-arm64") diff --git a/gradle.properties b/gradle.properties index 9eb0002da0..bbf6111e16 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,8 +1,9 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=2g -XX:MaxHeapSize=4g org.gradle.caching=true -kotlinVersion=1.9.0 -dokkaVersion=1.8.20 -scijavaParentPOMVersion=35.1.1 -lwjglVersion=3.3.2 -jvmTarget=11 +kotlinVersion=1.9.23 +dokkaVersion=1.9.10 +scijavaParentPOMVersion=37.0.0 +lwjglVersion=3.3.3 +jvmTarget=21 +version=0.11.0 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a79e2..d64cd49177 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 7c2c8c24b3..381baa9cef 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=591855b517fc635b9e04de1d05d5e76ada3f89f5fc76f87978d1b245b4f69225 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip +distributionSha256Sum=544c35d6bd849ae8a5ed0bcea39ba677dc40f49df7d1835561582da2009b961d +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index aeb74cbb43..1aa94a4269 100755 --- a/gradlew +++ b/gradlew @@ -83,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -130,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -141,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -149,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -198,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/gradlew.bat b/gradlew.bat index 6689b85bee..7101f8e467 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -43,11 +43,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail diff --git a/jitpack.yml b/jitpack.yml index 445e811f77..8f11677267 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -1,3 +1,3 @@ before_install: - - sdk install java 11.0.16-tem - - sdk use java 11.0.16-tem + - sdk install java 21.0.2-zulu + - sdk use java 21.0.2-zulu diff --git a/settings.gradle.kts b/settings.gradle.kts index 9df5410978..63cc891245 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -32,6 +32,6 @@ rootProject.name = "scenery" gradle.rootProject { group = "graphics.scenery" - version = "0.9.0" + version = project.properties["version"]!! description = "flexible scenegraphing and rendering for scientific visualisation" } diff --git a/src/main/kotlin/graphics/scenery/AmbientLight.kt b/src/main/kotlin/graphics/scenery/AmbientLight.kt index c6b4ca8cab..9b6bb759cc 100644 --- a/src/main/kotlin/graphics/scenery/AmbientLight.kt +++ b/src/main/kotlin/graphics/scenery/AmbientLight.kt @@ -83,7 +83,7 @@ class AmbientLight(intensity: Float = 1.0f, emissionColor: Vector3f = Vector3f(1 blending.destinationAlphaBlendFactor = Blending.BlendFactor.One blending.alphaBlending = Blending.BlendOp.add cullingMode = Material.CullingMode.Front - depthTest = Material.DepthTest.Greater + depthOp = Material.DepthTest.Greater } } } diff --git a/src/main/kotlin/graphics/scenery/DirectionalLight.kt b/src/main/kotlin/graphics/scenery/DirectionalLight.kt index 140e63e346..6dd4185186 100644 --- a/src/main/kotlin/graphics/scenery/DirectionalLight.kt +++ b/src/main/kotlin/graphics/scenery/DirectionalLight.kt @@ -81,7 +81,7 @@ class DirectionalLight(var direction: Vector3f = Vector3f(0.0f, 1.0f, 0.0f)) : L blending.destinationAlphaBlendFactor = Blending.BlendFactor.One blending.alphaBlending = Blending.BlendOp.add cullingMode = Material.CullingMode.Front - depthTest = Material.DepthTest.Greater + depthOp = Material.DepthTest.Greater } } } diff --git a/src/main/kotlin/graphics/scenery/FullscreenObject.kt b/src/main/kotlin/graphics/scenery/FullscreenObject.kt index 3e79b79706..3453423faa 100644 --- a/src/main/kotlin/graphics/scenery/FullscreenObject.kt +++ b/src/main/kotlin/graphics/scenery/FullscreenObject.kt @@ -9,6 +9,9 @@ import graphics.scenery.attribute.material.Material * @author Ulrik Günther */ class FullscreenObject : Mesh("FullscreenObject") { + + var wantsSync = false + override fun wantsSync(): Boolean = wantsSync init { geometry { // fake geometry diff --git a/src/main/kotlin/graphics/scenery/Mesh.kt b/src/main/kotlin/graphics/scenery/Mesh.kt index 12c34b1ef6..7235c691a8 100644 --- a/src/main/kotlin/graphics/scenery/Mesh.kt +++ b/src/main/kotlin/graphics/scenery/Mesh.kt @@ -222,7 +222,7 @@ open class Mesh(override var name: String = "Mesh") : DefaultNode(name), HasRend * @param[importMaterials] Whether a accompanying MTL file shall be used, defaults to true. * @param[flipNormals] Whether to flip the normals after reading them. */ - fun readFromOBJ(filename: String, importMaterials: Boolean = true, flipNormals: Boolean = false, useObjGroups: Boolean = true): Mesh { + fun readFromOBJ(filename: String, importMaterials: Boolean = true, flipNormals: Boolean = false, useGroupHeuristic: Boolean = true): Mesh { val logger by lazyLogger() var name = "" @@ -238,11 +238,7 @@ open class Mesh(override var name: String = "Mesh") : DefaultNode(name), HasRend var materials = HashMap() val normalSign = if(flipNormals) { -1.0f } else { 1.0f } - val groupDelimiter = if(useObjGroups) { - 'g' - } else { - 'o' - } + var groupDelimiter = 'g' /** * Recalculates normals, assuming CCW winding order. @@ -300,6 +296,23 @@ open class Mesh(override var name: String = "Mesh") : DefaultNode(name), HasRend val vertexCountMap = HashMap(50) val faceCountMap = HashMap(50) + var g = 0 + var o = 0 + Files.lines(p).forEach {line -> + val tokens = line.trim().trimEnd() + if(tokens.isNotEmpty()) { + when(tokens[0]) { + 'g' -> g++ + 'o' -> o++ + } + } + } + + if(o > 0 && g > 0 && useGroupHeuristic) { + logger.info("Using g/o heuristic for groups in OBJ file. Set useGroupHeuristic = false to disable.") + groupDelimiter = 'o' + } + val preparseStart = System.nanoTime() logger.info("Starting preparse") lines.forEach { @@ -338,7 +351,7 @@ open class Mesh(override var name: String = "Mesh") : DefaultNode(name), HasRend val indexBuffers = HashMap>() val faceBuffers = HashMap>() - vertexCountMap.forEach { objectName, objectVertexCount -> + vertexCountMap.forEach { (objectName, objectVertexCount) -> vertexBuffers[objectName] = Triple( MemoryUtil.memAlloc(objectVertexCount * meshGeometry.vertexSize * 4).order(ByteOrder.nativeOrder()).asFloatBuffer(), MemoryUtil.memAlloc(objectVertexCount * meshGeometry.vertexSize * 4).order(ByteOrder.nativeOrder()).asFloatBuffer(), diff --git a/src/main/kotlin/graphics/scenery/PointLight.kt b/src/main/kotlin/graphics/scenery/PointLight.kt index b9e5506c38..83e0ab25b5 100644 --- a/src/main/kotlin/graphics/scenery/PointLight.kt +++ b/src/main/kotlin/graphics/scenery/PointLight.kt @@ -15,8 +15,8 @@ import org.joml.Vector4f * @author Ulrik Günther * @constructor Creates a PointLight with default settings, e.g. white emission color. */ -class PointLight(val radius: Float = 5.0f) : Light("PointLight") { - private var proxySphere = Sphere(radius * 1.1f, 10) +open class PointLight(val radius: Float = 5.0f) : Light("PointLight") { + private var proxySphere = Sphere(radius * 2.0f, 10) /** The intensity of the point light. Bound to [0.0, 1.0] if using non-HDR rendering. */ @ShaderProperty override var intensity: Float = 1.0f @@ -33,7 +33,7 @@ class PointLight(val radius: Float = 5.0f) : Light("PointLight") { @ShaderProperty var lightRadius: Float = radius set(value) { if(value != lightRadius) { - logger.info("Resetting light radius") + logger.debug("Resetting light radius") field = value proxySphere = Sphere(value * 1.1f, 10) geometry { @@ -86,7 +86,7 @@ class PointLight(val radius: Float = 5.0f) : Light("PointLight") { blending.destinationAlphaBlendFactor = Blending.BlendFactor.One blending.alphaBlending = Blending.BlendOp.add cullingMode = Material.CullingMode.Front - depthTest = Material.DepthTest.Greater + depthOp = Material.DepthTest.Greater } } diff --git a/src/main/kotlin/graphics/scenery/RichNode.kt b/src/main/kotlin/graphics/scenery/RichNode.kt index cf9f936ec7..fe433cafd0 100644 --- a/src/main/kotlin/graphics/scenery/RichNode.kt +++ b/src/main/kotlin/graphics/scenery/RichNode.kt @@ -3,6 +3,9 @@ package graphics.scenery import graphics.scenery.attribute.renderable.HasRenderable import graphics.scenery.attribute.material.HasMaterial import graphics.scenery.attribute.spatial.HasSpatial +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.Volume.Companion.fromPathRawSplit +import org.joml.Vector3f open class RichNode(override var name: String = "Node") : DefaultNode (name), HasRenderable, HasMaterial, HasSpatial { init { diff --git a/src/main/kotlin/graphics/scenery/attribute/material/DefaultMaterial.kt b/src/main/kotlin/graphics/scenery/attribute/material/DefaultMaterial.kt index 9407ad87ff..71bbf9ab52 100644 --- a/src/main/kotlin/graphics/scenery/attribute/material/DefaultMaterial.kt +++ b/src/main/kotlin/graphics/scenery/attribute/material/DefaultMaterial.kt @@ -5,6 +5,7 @@ import graphics.scenery.net.Networkable import graphics.scenery.textures.Texture import graphics.scenery.utils.TimestampedConcurrentHashMap import org.joml.Vector3f +import org.joml.Vector4f import kotlin.reflect.KClass open class DefaultMaterial : Material, Networkable { @@ -35,12 +36,19 @@ open class DefaultMaterial : Material, Networkable { field = value updateModifiedAt() } + override var emissive: Vector4f = Vector4f(0.0f, 0.0f, 0.0f, 0.0f) + set(value) { + field = value + updateModifiedAt() + } override var blending: Blending = Blending() @Volatile @Transient override var textures: TimestampedConcurrentHashMap = TimestampedConcurrentHashMap() override var cullingMode: Material.CullingMode = Material.CullingMode.Back - override var depthTest: Material.DepthTest = Material.DepthTest.LessEqual + override var depthTest: Boolean = true + override var depthWrite: Boolean = true + override var depthOp: Material.DepthTest = Material.DepthTest.LessEqual override var wireframe: Boolean = false override var timestamp: Long = System.nanoTime() override var modifiedAt = Long.MIN_VALUE diff --git a/src/main/kotlin/graphics/scenery/attribute/material/Material.kt b/src/main/kotlin/graphics/scenery/attribute/material/Material.kt index 37b3a6c2cb..8f6a369af7 100644 --- a/src/main/kotlin/graphics/scenery/attribute/material/Material.kt +++ b/src/main/kotlin/graphics/scenery/attribute/material/Material.kt @@ -5,6 +5,7 @@ import graphics.scenery.attribute.material.Material.CullingMode.* import graphics.scenery.textures.Texture import graphics.scenery.utils.TimestampedConcurrentHashMap import org.joml.Vector3f +import org.joml.Vector4f /** * Material interface, storing material colors, textures, opacity properties, etc. @@ -37,6 +38,8 @@ interface Material { var roughness: Float /** Metallicity, 0.0 is non-metal, 1.0 is full metal */ var metallic: Float + /** Emission of the material and corresponding strength */ + var emissive: Vector4f /** Blending settings for this material. See [Blending]. */ var blending: Blending @@ -49,8 +52,14 @@ interface Material { /** Culling mode of the material. @see[CullingMode] */ var cullingMode: CullingMode + /** Enable or disable depth testing */ + var depthTest: Boolean + + /** Enable or disable writing to the depth buffer */ + var depthWrite: Boolean + /** depth testing mode for this material */ - var depthTest: DepthTest + var depthOp: DepthTest /** Flag to make the object wireframe */ var wireframe: Boolean @@ -66,9 +75,11 @@ interface Material { fun materialHashCode() : Int { var result = blending.hashCode() result = 31 * result + cullingMode.hashCode() - result = 31 * result + depthTest.hashCode() + result = 31 * result + depthOp.hashCode() result = 31 * result + wireframe.hashCode() result = 31 * result + timestamp.hashCode() + result = 31 * result + depthTest.hashCode() + result = 31 * result + depthWrite.hashCode() return result } } diff --git a/src/main/kotlin/graphics/scenery/attribute/spatial/DefaultSpatial.kt b/src/main/kotlin/graphics/scenery/attribute/spatial/DefaultSpatial.kt index 3341ef5bd8..422b7c87ff 100644 --- a/src/main/kotlin/graphics/scenery/attribute/spatial/DefaultSpatial.kt +++ b/src/main/kotlin/graphics/scenery/attribute/spatial/DefaultSpatial.kt @@ -1,5 +1,6 @@ package graphics.scenery.attribute.spatial +import graphics.scenery.Camera import graphics.scenery.DefaultNode import graphics.scenery.Node import graphics.scenery.Scene @@ -12,6 +13,7 @@ import net.imglib2.RealLocalizable import org.joml.* import java.lang.Float.max import java.lang.Float.min +import java.lang.Math import kotlin.reflect.KClass import kotlin.reflect.KProperty @@ -178,6 +180,35 @@ open class DefaultSpatial(@Transient protected var node: Node = DefaultNode()) : return center } + /** + * This function rotates this spatial by a fixed [yaw] and [pitch] about the [target] + * + * @param[yaw] yaw in degrees + * @param[pitch] pitch in degrees + * @param[target] the target position + */ + override fun rotateAround(yaw: Float, pitch: Float, target: Vector3f) { + val frameYaw = (yaw) / 180.0f * Math.PI.toFloat() + val framePitch = pitch / 180.0f * Math.PI.toFloat() + + // first calculate the total rotation quaternion to be applied to the camera + val yawQ = Quaternionf().rotateXYZ(0.0f, frameYaw, 0.0f).normalize() + val pitchQ = Quaternionf().rotateXYZ(framePitch, 0.0f, 0.0f).normalize() + + node.ifSpatial { + val distance = (target - position).length() + rotation = pitchQ.mul(rotation).mul(yawQ).normalize() + if(node is Camera) { + position = target + (node as Camera).forward * distance * (-1.0f) + (node as Camera).target = target + } else { + val forward = world.transform(Vector4f(1.0f, 0.0f, 0.0f, 1.0f)).xyz() + position = target + forward * distance * (-1.0f) + } + } + } + + override fun putAbove(position: Vector3f): Vector3f { val center = centerOn(position) diff --git a/src/main/kotlin/graphics/scenery/attribute/spatial/Spatial.kt b/src/main/kotlin/graphics/scenery/attribute/spatial/Spatial.kt index b0e9dfca4f..b07558ea5b 100644 --- a/src/main/kotlin/graphics/scenery/attribute/spatial/Spatial.kt +++ b/src/main/kotlin/graphics/scenery/attribute/spatial/Spatial.kt @@ -107,6 +107,15 @@ interface Spatial: RealLocalizable, RealPositionable, Networkable { */ fun centerOn(position: Vector3f): Vector3f + /** + * This function rotates this spatial by a fixed [yaw] and [pitch] about the [target] + * + * @param[yaw] yaw in degrees + * @param[pitch] pitch in degrees + * @param[target] the target position + */ + fun rotateAround(yaw: Float, pitch: Float, target: Vector3f) + /** * Orients the Node between points [p1] and [p2], and optionally * [rescale]s and [reposition]s it. diff --git a/src/main/kotlin/graphics/scenery/backends/Display.kt b/src/main/kotlin/graphics/scenery/backends/Display.kt index 637488e468..9fc1f2d558 100644 --- a/src/main/kotlin/graphics/scenery/backends/Display.kt +++ b/src/main/kotlin/graphics/scenery/backends/Display.kt @@ -60,7 +60,7 @@ interface Display { */ fun submitToCompositorVulkan(width: Int, height: Int, format: Int, instance: VkInstance, device: VulkanDevice, - queue: VkQueue, + queueWithMutex: VulkanDevice.QueueWithMutex, image: Long) /** diff --git a/src/main/kotlin/graphics/scenery/backends/RenderConfigReader.kt b/src/main/kotlin/graphics/scenery/backends/RenderConfigReader.kt index 14eb77f6bd..06de2c9c48 100644 --- a/src/main/kotlin/graphics/scenery/backends/RenderConfigReader.kt +++ b/src/main/kotlin/graphics/scenery/backends/RenderConfigReader.kt @@ -3,7 +3,9 @@ package graphics.scenery.backends import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.fasterxml.jackson.module.kotlin.SingletonSupport import graphics.scenery.Blending import graphics.scenery.utils.JsonDeserialisers import graphics.scenery.utils.lazyLogger @@ -93,7 +95,7 @@ class RenderConfigReader { */ data class RendertargetConfig( @JsonDeserialize(using = JsonDeserialisers.FloatPairDeserializer::class) var size: Pair = Pair(1.0f, 1.0f), - val attachments: Map = emptyMap() + val attachments: LinkedHashMap = LinkedHashMap() ) data class RendertargetBinding( @@ -178,7 +180,16 @@ class RenderConfigReader { */ fun loadFromFile(path: String): RenderConfig { val mapper = ObjectMapper(YAMLFactory()) - mapper.registerModule(KotlinModule()) + mapper.registerModule( + KotlinModule.Builder() + .withReflectionCacheSize(512) + .configure(KotlinFeature.NullToEmptyCollection, true) + .configure(KotlinFeature.NullToEmptyMap, true) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, true) + .configure(KotlinFeature.StrictNullChecks, true) + .build() + ) var stream = this.javaClass.getResourceAsStream(path) diff --git a/src/main/kotlin/graphics/scenery/backends/Renderer.kt b/src/main/kotlin/graphics/scenery/backends/Renderer.kt index 75c91db9f7..6b22e22f81 100644 --- a/src/main/kotlin/graphics/scenery/backends/Renderer.kt +++ b/src/main/kotlin/graphics/scenery/backends/Renderer.kt @@ -4,8 +4,8 @@ import graphics.scenery.* import graphics.scenery.backends.vulkan.VulkanRenderer import graphics.scenery.textures.Texture import graphics.scenery.utils.ExtractsNatives -import graphics.scenery.utils.lazyLogger import graphics.scenery.utils.SceneryPanel +import graphics.scenery.utils.lazyLogger import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.selects.select @@ -40,6 +40,10 @@ abstract class Renderer : Hubable { abstract var firstImageReady: Boolean protected set + /** The total number of frames rendered so far. */ + var totalFrames = 0L + protected set + /** [Settings] instance the renderer is using. */ abstract var settings: Settings @@ -175,6 +179,11 @@ abstract class Renderer : Hubable { @Volatile protected var textureRequests = ConcurrentLinkedQueue>>() + /** + * A list of user-defined lambdas that will be executed once per iteration of the render loop + */ + val runAfterRendering = ArrayList<()->Unit>() + /** * Requests the renderer to update [texture]'s contents from the GPU. [onReceive] is executed * on receiving the data. diff --git a/src/main/kotlin/graphics/scenery/backends/ShaderCompiler.kt b/src/main/kotlin/graphics/scenery/backends/ShaderCompiler.kt index d1ff58c851..5320c8c801 100644 --- a/src/main/kotlin/graphics/scenery/backends/ShaderCompiler.kt +++ b/src/main/kotlin/graphics/scenery/backends/ShaderCompiler.kt @@ -1,6 +1,7 @@ package graphics.scenery.backends import graphics.scenery.utils.lazyLogger +import org.lwjgl.system.MemoryUtil import org.lwjgl.util.shaderc.Shaderc import org.slf4j.Logger import picocli.CommandLine diff --git a/src/main/kotlin/graphics/scenery/backends/ShaderPackage.kt b/src/main/kotlin/graphics/scenery/backends/ShaderPackage.kt index bf6d57b08f..1e23ad3bc1 100644 --- a/src/main/kotlin/graphics/scenery/backends/ShaderPackage.kt +++ b/src/main/kotlin/graphics/scenery/backends/ShaderPackage.kt @@ -20,7 +20,8 @@ data class ShaderPackage(val baseClass: Class<*>, val codePath: String?, val spirv: ByteArray?, val code: String?, - var priority: SourceSPIRVPriority) { + var priority: SourceSPIRVPriority, + val disableCaching: Boolean = false) { private val logger by lazyLogger() init { diff --git a/src/main/kotlin/graphics/scenery/backends/Shaders.kt b/src/main/kotlin/graphics/scenery/backends/Shaders.kt index 8f3891e1cd..496471ea7a 100644 --- a/src/main/kotlin/graphics/scenery/backends/Shaders.kt +++ b/src/main/kotlin/graphics/scenery/backends/Shaders.kt @@ -1,5 +1,6 @@ package graphics.scenery.backends +import bvv.core.shadergen.Shader import graphics.scenery.utils.lazyLogger import kotlinx.coroutines.* import java.io.File @@ -48,7 +49,9 @@ sealed class Shaders() { * [shaders], which are assumed to be relative to a class [clazz]. */ open class ShadersFromFiles(val shaders: Array, - val clazz: Class<*> = Renderer::class.java) : Shaders() { + val clazz: Class<*> = Renderer::class.java) : Shaders() { + + init { type.addAll(shaders.map { val extension = it.lowercase().substringBeforeLast(".spv").substringAfterLast(".").trim() @@ -89,33 +92,48 @@ sealed class Shaders() { val baseClass = arrayOf(spirvPath, codePath).mapNotNull { safeFindBaseClass(arrayOf(clazz, Renderer::class.java), it) } - if (baseClass.isEmpty()) { + val fileCandidates = listOf(File(codePath), File("shaders/$codePath")) + val f = fileCandidates.firstOrNull { it.exists() } + val paths = ShaderPaths(spirvPath, codePath) + var disallowCaching = false + + if (baseClass.isEmpty() && f == null) { throw ShaderCompilationException("Shader files for $shaderCodePath ($spirvPath, $codePath) not found.") } val base = baseClass.first() val pathPrefix = base.second - - val localFiles = listOf(File(codePath), File("shaders/$codePath")) - val paths = ShaderPaths(spirvPath, codePath) - val f = localFiles.find { it.exists() } + var shaderPackage: ShaderPackage? = null val (spirvFromFile, codeFromFile) = if(f != null && f.exists()) { - logger.warn("Using $codePath from filesystem, will ignore compiled or packaged version.") - changeTimes.putIfAbsent(ShaderPaths(spirvPath, codePath), f.lastModified()) + logger.warn("Using $codePath from filesystem (${f.absolutePath}), will ignore compiled or packaged version.") - CoroutineScope(Dispatchers.IO).launch { - while(watchFiles) { - val modifiedTime = f.lastModified() - if(modifiedTime > (changeTimes[paths] ?: 0)) { - logger.info("File changed, marking $codePath as stale") - changeTimes[paths] = modifiedTime - stale = true - watchFiles = false + changeTimes.putIfAbsent(ShaderPaths(spirvPath, codePath), f.lastModified()) + disallowCaching = true + + watchers.putIfAbsent(paths, + CoroutineScope(Dispatchers.IO).launch { + while(isActive) { + val modifiedTime = f.lastModified() + val sp = shaderPackage + if(modifiedTime > (changeTimes[paths] ?: 0) && sp != null) { + // try compiling the changed shader, bail if this fails to avoid + // replacing a working shader with a broken one + try { + val newShaderPackage = sp.copy(code = f.readText()) + compile(newShaderPackage, type, target, base.first) + + logger.info("File changed and test compilation succeeded, marking $codePath as stale") + changeTimes[paths] = modifiedTime + stale = true + } catch(e: ShaderCompilationException) { + logger.error("Compilation error during hot reloading, will not replace shader") + } + } + + delay(2000.milliseconds) } - delay(2000.milliseconds) - } - } + }) null to f.readText() } else { @@ -123,16 +141,17 @@ sealed class Shaders() { base.first.getResourceAsStream("$pathPrefix$codePath")?.bufferedReader().use { it?.readText() } } - val shaderPackage = ShaderPackage(base.first, + shaderPackage = ShaderPackage(base.first, type, "$pathPrefix$spirvPath", "$pathPrefix$codePath", spirvFromFile, codeFromFile, - SourceSPIRVPriority.SourcePriority) + SourceSPIRVPriority.SourcePriority, + disallowCaching) val p = compile(shaderPackage, type, target, base.first) - if(stale) { + if(stale && !disallowCaching) { cache[paths] = p } stale = false @@ -156,6 +175,7 @@ sealed class Shaders() { companion object { protected val cache = ConcurrentHashMap() protected val changeTimes = ConcurrentHashMap() + protected var watchers = ConcurrentHashMap() } } diff --git a/src/main/kotlin/graphics/scenery/backends/UBO.kt b/src/main/kotlin/graphics/scenery/backends/UBO.kt index f81bb1756b..959655aeeb 100644 --- a/src/main/kotlin/graphics/scenery/backends/UBO.kt +++ b/src/main/kotlin/graphics/scenery/backends/UBO.kt @@ -351,12 +351,22 @@ open class UBO { * * Takes into consideration the member's name and _invoked_ value, as well as the * buffer's memory address to discern buffer switches the UBO is oblivious to. + * + * In case the member's _invoked_ value is an IntArray or FloatArray, contentHashCode() + * is used instead of hashCode(), such that simple array reallocation without content + * changes do not affect the hashing outcome. */ protected fun getMembersHash(buffer: ByteBuffer): Int { - return members.map { (it.key.hashCode() xor it.value.invoke().hashCode()).toLong() } + return members.map { (it.key.hashCode() xor it.value.invoke().hash()).toLong() } .fold(31L) { acc, value -> acc + (value xor (value.ushr(32)))}.toInt() + MemoryUtil.memAddress(buffer).hashCode() } + private fun Any.hash(): Int = when(this) { + is FloatArray -> this.contentHashCode() + is IntArray -> this.contentHashCode() + else -> this.hashCode() + } + /** * Updates the currently stored member hash. */ diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/HeadlessSwapchain.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/HeadlessSwapchain.kt index 3f2976a3de..11de7408ec 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/HeadlessSwapchain.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/HeadlessSwapchain.kt @@ -20,7 +20,7 @@ import java.nio.LongBuffer * @author Ulrik Günther */ open class HeadlessSwapchain(device: VulkanDevice, - queue: VkQueue, + queue: VulkanDevice.QueueWithMutex, commandPools: VulkanRenderer.CommandPools, renderConfig: RenderConfigReader.RenderConfig, useSRGB: Boolean = true, @@ -114,23 +114,26 @@ open class HeadlessSwapchain(device: VulkanDevice, } else { VK10.VK_FORMAT_B8G8R8A8_UNORM } - presentQueue = VU.createDeviceQueue(device, device.queues.graphicsQueue.first) - - val textureImages = (0 until bufferCount).map { - val t = VulkanTexture(device, commandPools, queue, queue, window.width, window.height, 1, - format, 1) - val image = t.createImage(window.width, window.height, 1, format, + presentQueue = device.getQueue(device.queueIndices.graphicsQueue.first) + + val vulkanImages = (0 until bufferCount).map { + VulkanImage.create( + device, + window.width, + window.height, + 1, + format, VK10.VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT or VK10.VK_IMAGE_USAGE_TRANSFER_SRC_BIT or VK10.VK_IMAGE_USAGE_SAMPLED_BIT, - VK10.VK_IMAGE_TILING_OPTIMAL, VK10.VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 1) - t to image + VK10.VK_IMAGE_TILING_OPTIMAL, + VK10.VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + 1 + ) } - images = textureImages.map { - it.second.image - }.toLongArray() + images = vulkanImages.map { it.image }.toLongArray() - imageViews = textureImages.map { - it.first.createImageView(it.second, format) + imageViews = vulkanImages.map { + it.createView() }.toLongArray() val fenceCreateInfo = VkFenceCreateInfo.calloc() @@ -174,7 +177,7 @@ open class HeadlessSwapchain(device: VulkanDevice, */ override fun next(timeout: Long): Pair? { MemoryStack.stackPush().use { stack -> - VK10.vkQueueWaitIdle(presentQueue) + VK10.vkQueueWaitIdle(presentQueue.queue) val signal = stack.mallocLong(1) signal.put(0, imageAvailableSemaphores[currentImage]) @@ -257,7 +260,7 @@ open class HeadlessSwapchain(device: VulkanDevice, flush = true, dealloc = true) } - VK10.vkQueueWaitIdle(queue) + VK10.vkQueueWaitIdle(queue.queue) resizeHandler.queryResize() currentImage = (currentImage + 1) % images.size @@ -292,7 +295,7 @@ open class HeadlessSwapchain(device: VulkanDevice, * Closes the swapchain, deallocating all resources. */ override fun close() { - vkQueueWaitIdle(queue) + vkQueueWaitIdle(queue.queue) presentInfo.free() MemoryUtil.memFree(swapchainImage) diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/OpenGLSwapchain.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/OpenGLSwapchain.kt index c5bbaa802b..76d06a1a0b 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/OpenGLSwapchain.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/OpenGLSwapchain.kt @@ -37,7 +37,7 @@ import java.nio.LongBuffer * @author Ulrik Günther */ class OpenGLSwapchain(device: VulkanDevice, - queue: VkQueue, + queue: VulkanDevice.QueueWithMutex, commandPools: VulkanRenderer.CommandPools, renderConfig: RenderConfigReader.RenderConfig, useSRGB: Boolean = true, @@ -180,15 +180,12 @@ class OpenGLSwapchain(device: VulkanDevice, val fenceCreateInfo = VkFenceCreateInfo.calloc() .sType(VK10.VK_STRUCTURE_TYPE_FENCE_CREATE_INFO) - presentQueue = VU.createDeviceQueue(device, device.queues.graphicsQueue.first) + presentQueue = device.getQueue(device.queueIndices.graphicsQueue.first) val imgs = (0 until bufferCount).map { with(VU.newCommandBuffer(device, commandPools.Standard, autostart = true)) { - val t = VulkanTexture(this@OpenGLSwapchain.device, commandPools, queue, queue, - windowWidth, window.height, 1, format, 1) - - val image = t.createImage(windowWidth, window.height, 1, + val image = VulkanImage.create(this@OpenGLSwapchain.device, windowWidth, window.height, 1, format, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT or VK_IMAGE_USAGE_SAMPLED_BIT, VK_IMAGE_TILING_OPTIMAL, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 1) @@ -198,7 +195,7 @@ class OpenGLSwapchain(device: VulkanDevice, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 1, commandBuffer = this) - val view = t.createImageView(image, format) + val view = image.createView() imageAvailableSemaphores.add(this@OpenGLSwapchain.device.createSemaphore()) imageRenderedSemaphores.add(this@OpenGLSwapchain.device.createSemaphore()) @@ -398,7 +395,7 @@ class OpenGLSwapchain(device: VulkanDevice, // NVDrawVulkanImage.glSignalVkSemaphoreNV(-1L) // return (presentedFrames % 2) to -1L//.toInt() MemoryStack.stackPush().use { stack -> - VK10.vkQueueWaitIdle(queue) + VK10.vkQueueWaitIdle(queue.queue) val signal = stack.mallocLong(1) signal.put(0, imageAvailableSemaphores[currentImage]) @@ -491,7 +488,7 @@ class OpenGLSwapchain(device: VulkanDevice, throw IllegalStateException("Cannot use a window of type ${window.javaClass.simpleName}") } - vkQueueWaitIdle(queue) + vkQueueWaitIdle(queue.queue) closeSyncPrimitives() diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/SwingSwapchain.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/SwingSwapchain.kt index d89b9c12c8..ddad2168f5 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/SwingSwapchain.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/SwingSwapchain.kt @@ -11,7 +11,6 @@ import org.lwjgl.system.MemoryUtil.memFree import org.lwjgl.system.Platform import org.lwjgl.vulkan.KHRSurface import org.lwjgl.vulkan.KHRSwapchain -import org.lwjgl.vulkan.VkQueue import org.lwjgl.vulkan.awt.AWTVK import java.awt.BorderLayout import java.awt.Canvas @@ -34,7 +33,7 @@ import javax.swing.SwingUtilities * @author Ulrik Günther */ open class SwingSwapchain(override val device: VulkanDevice, - override val queue: VkQueue, + override val queue: VulkanDevice.QueueWithMutex, override val commandPools: VulkanRenderer.CommandPools, override val renderConfig: RenderConfigReader.RenderConfig, override val useSRGB: Boolean = true, @@ -180,7 +179,7 @@ open class SwingSwapchain(override val device: VulkanDevice, val mainFrame = JFrame(applicationName) mainFrame.layout = BorderLayout() - val sceneryPanel = SceneryJPanel() + val sceneryPanel = SceneryJPanel(owned = true) sceneryPanel.preferredSize = Dimension(windowWidth, windowHeight) mainFrame.add(sceneryPanel, BorderLayout.CENTER) mainFrame.pack() diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VU.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VU.kt index a76054e81b..6e582fd678 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VU.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VU.kt @@ -56,7 +56,7 @@ fun VkCommandBuffer.endCommandBuffer() { * the submission process. */ fun VkCommandBuffer.endCommandBuffer(device: VulkanDevice, commandPool: Long, - queue: VkQueue?, flush: Boolean = true, + queue: VulkanDevice.QueueWithMutex, flush: Boolean = true, dealloc: Boolean = false, submitInfoPNext: Pointer? = null, signalSemaphores: LongBuffer? = null, waitSemaphores: LongBuffer? = null, @@ -70,7 +70,7 @@ fun VkCommandBuffer.endCommandBuffer(device: VulkanDevice, commandPool: Long, throw AssertionError("Failed to end command buffer $this") } - if (flush && queue != null) { + if (flush) { this.submit(queue, submitInfoPNext, waitSemaphores = waitSemaphores, signalSemaphores = signalSemaphores, waitDstStageMask = waitDstStageMask, block = block, fence = fence) } @@ -85,7 +85,7 @@ fun VkCommandBuffer.endCommandBuffer(device: VulkanDevice, commandPool: Long, * [submitInfoPNext], [signalSemaphores], [waitSemaphores] and [waitDstStageMask] can be used to further fine-grain * the submission process. */ -fun VkCommandBuffer.submit(queue: VkQueue, submitInfoPNext: Pointer? = null, +fun VkCommandBuffer.submit(queue: VulkanDevice.QueueWithMutex, submitInfoPNext: Pointer? = null, signalSemaphores: LongBuffer? = null, waitSemaphores: LongBuffer? = null, waitDstStageMask: IntBuffer? = null, block: Boolean = true, fence: Long? = null) { @@ -100,7 +100,9 @@ fun VkCommandBuffer.submit(queue: VkQueue, submitInfoPNext: Pointer? = null, .pSignalSemaphores(signalSemaphores) .pNext(submitInfoPNext?.address() ?: NULL) - if(waitSemaphores?.remaining() ?: 0 > 0 && waitSemaphores != null && waitDstStageMask != null) { + if((waitSemaphores?.remaining() ?: 0) > 0 + && waitSemaphores != null + && waitDstStageMask != null) { submitInfo .waitSemaphoreCount(waitSemaphores.remaining()) .pWaitSemaphores(waitSemaphores) @@ -108,10 +110,16 @@ fun VkCommandBuffer.submit(queue: VkQueue, submitInfoPNext: Pointer? = null, } if(block) { - vkQueueSubmit(queue, submitInfo, fence ?: VK_NULL_HANDLE) - vkQueueWaitIdle(queue) + queue.mutex.acquire() + vkQueueSubmit(queue.queue, submitInfo, fence ?: VK_NULL_HANDLE) + val r = vkQueueWaitIdle(queue.queue) + queue.mutex.release() + r } else { - vkQueueSubmit(queue, submitInfo, fence ?: VK_NULL_HANDLE) + queue.mutex.acquire() + val r = vkQueueSubmit(queue.queue, submitInfo, fence ?: VK_NULL_HANDLE) + queue.mutex.release() + r } }, { }) } @@ -498,16 +506,6 @@ class VU { } } - /** - * Creates a new Vulkan queue on [device] with the queue family index [queueFamilyIndex] and returns the queue. - */ - fun createDeviceQueue(device: VulkanDevice, queueFamilyIndex: Int): VkQueue { - val queue = getPointer("Getting device queue for queueFamilyIndex=$queueFamilyIndex", - { vkGetDeviceQueue(device.vulkanDevice, queueFamilyIndex, 0, this); VK_SUCCESS }, {}) - - return VkQueue(queue, device.vulkanDevice) - } - /** * Creates and returns a new command buffer on [device], associated with [commandPool]. By default, it'll be a primary * command buffer, that can be changed by setting [level] to [VK_COMMAND_BUFFER_LEVEL_SECONDARY]. diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanCommandBuffer.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanCommandBuffer.kt index fac866806c..e930d15ca9 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanCommandBuffer.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanCommandBuffer.kt @@ -84,12 +84,12 @@ class VulkanCommandBuffer(val device: VulkanDevice, var commandBuffer: VkCommand 0, 2, timingArray, - 0, + 8, VK_QUERY_RESULT_64_BIT ) }) - val validBits = device.queues.graphicsQueue.second.timestampValidBits() + val validBits = device.queueIndices.graphicsQueue.second.timestampValidBits() runtime = (keepBits(timingArray[1], validBits) - keepBits( timingArray[0], validBits diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanComputePass.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanComputePass.kt index 43212e187f..950cb0a5fb 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanComputePass.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanComputePass.kt @@ -9,6 +9,7 @@ import org.joml.Vector3i import org.lwjgl.system.MemoryUtil import org.lwjgl.vulkan.KHRSwapchain import org.lwjgl.vulkan.VK10 +import kotlin.math.ceil /** * Helper object for compute pass command buffer recording. @@ -92,9 +93,9 @@ object VulkanComputePass { } VK10.vkCmdDispatch(this, - metadata.workSizes.x() / maxOf(localSizes.first, 1), - metadata.workSizes.y() / maxOf(localSizes.second, 1), - metadata.workSizes.z() / maxOf(localSizes.third, 1)) + ceil(metadata.workSizes.x().toFloat() / maxOf(localSizes.first, 1).toFloat()).toInt(), + ceil(metadata.workSizes.y().toFloat() / maxOf(localSizes.second, 1).toFloat()).toInt(), + ceil(metadata.workSizes.z().toFloat() / maxOf(localSizes.third, 1).toFloat()).toInt()) loadStoreAttachments .forEach { (isOutput, fb) -> diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanDevice.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanDevice.kt index 4db00eb9ff..25c5dc7dc2 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanDevice.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanDevice.kt @@ -14,6 +14,7 @@ import org.lwjgl.vulkan.VK11.VK_FORMAT_G8B8G8R8_422_UNORM import java.nio.ByteBuffer import java.util.* import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.Semaphore typealias QueueIndexWithProperties = Pair @@ -38,11 +39,14 @@ open class VulkanDevice( /** Stores the Vulkan-internal device. */ val vulkanDevice: VkDevice /** Stores available queue indices. */ - val queues: Queues + val queueIndices: QueueIndices + private val queueProps: VkQueueFamilyProperties.Buffer /** Stores available extensions */ val extensions = ArrayList() /** Stores enabled features */ val features: VkPhysicalDeviceFeatures + /** Stores the created device queues. */ + private val queues = ConcurrentHashMap>() private val descriptorPools = ArrayList(5) @@ -51,6 +55,11 @@ open class VulkanDevice( */ enum class DeviceType { Unknown, Other, IntegratedGPU, DiscreteGPU, VirtualGPU, CPU } + /** + * Data class to hold a Vulkan queue together with a mutex (via a [Semaphore]) for regulating access. + */ + data class QueueWithMutex(val queue: VkQueue, val mutex: Semaphore = Semaphore(1)) + /** * Class to store device-specific metadata. * @@ -79,7 +88,7 @@ open class VulkanDevice( * @property[graphicsQueue] The index of the graphics queue * @property[computeQueue] The index of the compute queue */ - data class Queues(val presentQueue: QueueIndexWithProperties, val transferQueue: QueueIndexWithProperties, val graphicsQueue: QueueIndexWithProperties, val computeQueue: QueueIndexWithProperties) + data class QueueIndices(val presentQueue: QueueIndexWithProperties, val transferQueue: QueueIndexWithProperties, val graphicsQueue: QueueIndexWithProperties, val computeQueue: QueueIndexWithProperties) init { val enabledFeatures = VkPhysicalDeviceFeatures.calloc() @@ -87,38 +96,65 @@ open class VulkanDevice( val pQueueFamilyPropertyCount = stack.callocInt(1) vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, pQueueFamilyPropertyCount, null) val queueCount = pQueueFamilyPropertyCount.get(0) - val queueProps = VkQueueFamilyProperties.calloc(queueCount) + queueProps = VkQueueFamilyProperties.calloc(queueCount) vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, pQueueFamilyPropertyCount, queueProps) - var graphicsQueueFamilyIndex = 0 - var transferQueueFamilyIndex = 0 - var computeQueueFamilyIndex = 0 - val presentQueueFamilyIndex = 0 + var graphicsQueueFamilyIndex = -1 + var transferQueueFamilyIndex = -1 + var computeQueueFamilyIndex = -1 + var presentQueueFamilyIndex = -1 var index = 0 while (index < queueCount) { - if (queueProps.get(index).queueFlags() and VK_QUEUE_GRAPHICS_BIT != 0) { + if (queueProps.get(index).queueFlags() and VK_QUEUE_GRAPHICS_BIT != 0 && graphicsQueueFamilyIndex == -1) { graphicsQueueFamilyIndex = index + presentQueueFamilyIndex = index + index++ + continue } - if (queueProps.get(index).queueFlags() and VK_QUEUE_TRANSFER_BIT != 0) { + if (queueProps.get(index).queueFlags() and VK_QUEUE_TRANSFER_BIT != 0 && transferQueueFamilyIndex == -1) { transferQueueFamilyIndex = index + index++ + continue } - if (queueProps.get(index).queueFlags() and VK_QUEUE_COMPUTE_BIT != 0) { + if (queueProps.get(index).queueFlags() and VK_QUEUE_COMPUTE_BIT != 0 && computeQueueFamilyIndex == -1) { computeQueueFamilyIndex = index + index++ + continue + } + + // if transfer and compute families end up on the same index, let's see + // if we can find more. Otherwise, we revert to the same index. + if(transferQueueFamilyIndex == graphicsQueueFamilyIndex) { + transferQueueFamilyIndex = -1 + } + + if(computeQueueFamilyIndex == graphicsQueueFamilyIndex) { + computeQueueFamilyIndex = -1 } index++ } + // no distinct queues possible, revert to index 0 for the default queue + if(transferQueueFamilyIndex == -1) { + transferQueueFamilyIndex = graphicsQueueFamilyIndex + } + + if(computeQueueFamilyIndex == -1) { + computeQueueFamilyIndex = graphicsQueueFamilyIndex + } + val requiredFamilies = listOf( graphicsQueueFamilyIndex, + presentQueueFamilyIndex, transferQueueFamilyIndex, computeQueueFamilyIndex) .groupBy { it } - logger.info("Creating ${requiredFamilies.size} distinct queue groups") + logger.debug("Creating ${requiredFamilies.size} distinct queue groups") /** * Adjusts the queue count of a [VkDeviceQueueCreateInfo] struct to [num]. @@ -132,10 +168,10 @@ open class VulkanDevice( requiredFamilies.entries.forEachIndexed { i, (familyIndex, group) -> val size = minOf(queueProps.get(familyIndex).queueCount(), group.size) - logger.debug("Adding queue with familyIndex=$familyIndex, size=$size") + logger.debug("Adding queue with familyIndex=$familyIndex, size=$size (group size ${group.size})") val pQueuePriorities = stack.callocFloat(group.size) - for(pr in 0 until group.size) { pQueuePriorities.put(pr, 1.0f) } + for(pr in group.indices) { pQueuePriorities.put(pr, 1.0f) } queueCreateInfo[i] .sType(VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO) @@ -218,7 +254,7 @@ open class VulkanDevice( graphicsQueueFamilyIndex, computeQueueFamilyIndex, presentQueueFamilyIndex, transferQueueFamilyIndex, memoryProperties) Triple(VkDevice(device, physicalDevice, deviceCreateInfo), - Queues( + QueueIndices( presentQueue = presentQueueFamilyIndex to queueProps[presentQueueFamilyIndex], transferQueue = transferQueueFamilyIndex to queueProps[transferQueueFamilyIndex], computeQueue = computeQueueFamilyIndex to queueProps[computeQueueFamilyIndex], @@ -228,7 +264,7 @@ open class VulkanDevice( } vulkanDevice = result.first - queues = result.second + queueIndices = result.second memoryProperties = result.third features = enabledFeatures @@ -357,12 +393,76 @@ open class VulkanDevice( } } + /** + * Retrieves a Vulkan queue on [device] with the queue family index [queueFamilyIndex]. If none exists + * yet, one will be created. + */ + fun getQueue(queueFamilyIndex: Int): QueueWithMutex { + val queue = queues[queueFamilyIndex]?.last() ?: createQueue(queueFamilyIndex) + + return queue + } + + /* + * Creates a new Vulkan queue on [device] with the queue family index [queueFamilyIndex] and returns the queue. + */ + fun createQueue(queueFamilyIndex: Int): QueueWithMutex { + val index = queues[queueFamilyIndex]?.lastIndex ?: 0 + + val availableQueues = queueProps[queueFamilyIndex].queueCount() + + if(index + 1 > availableQueues) { + logger.warn("Don't have more queues available for index $queueFamilyIndex, returning existing queue.") + return queues[queueFamilyIndex]!!.last() + } + + logger.debug("Requesting device queue of family $queueFamilyIndex with index $index") + val q = VU.getPointer("Getting device queue for queueFamilyIndex=$queueFamilyIndex", + { vkGetDeviceQueue(vulkanDevice, queueFamilyIndex, index, this); VK_SUCCESS }, {}) + + val qm = QueueWithMutex(VkQueue(q, vulkanDevice)) + queues.getOrPut(queueFamilyIndex) { arrayListOf() }?.add(qm) + + return qm + } + + /** + * Creates a new fence on this device, returning the handle. + */ + fun createFence(): Long { + stackPush().use { stack -> + val fc = VkFenceCreateInfo.calloc(stack) + .sType(VK_STRUCTURE_TYPE_FENCE_CREATE_INFO) + .pNext(MemoryUtil.NULL) + + return VU.getLong("Creating fence", + { vkCreateFence(vulkanDevice, fc, null, this) }, {}) + } + } + + /** + * Destroys a given [fence] that resides on this device. + */ + fun destroyFence(fence: Long) { + vkDestroyFence(vulkanDevice, fence, null) + } + fun removeSemaphore(semaphore: Long) { logger.debug("Removing semaphore {}", semaphore.toHexString().lowercase()) vkDestroySemaphore(this.vulkanDevice, semaphore, null) } - data class DescriptorPool(val handle: Long, var free: Int = maxTextures + maxUBOs + maxInputAttachments + maxUBOs) { + data class DescriptorPool( + val handle: Long, + var free: ConcurrentHashMap = ConcurrentHashMap( + mutableMapOf( + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER to maxUBOs, + VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC to maxUBOs, + VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT to maxInputAttachments, + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER to maxTextures, + VK_DESCRIPTOR_TYPE_STORAGE_IMAGE to maxTextures) + ) + ) { companion object { val maxTextures = 2048 * 16 @@ -370,6 +470,10 @@ open class VulkanDevice( val maxInputAttachments = 32 val maxSets = maxUBOs * 2 + maxInputAttachments + maxTextures } + + fun decreaseAvailable(type: Int, by: Int = 1) { + free[type] = free[type]!! - by + } } private fun createDescriptorPool(): DescriptorPool { @@ -409,7 +513,7 @@ open class VulkanDevice( val handle = VU.getLong("vkCreateDescriptorPool", { vkCreateDescriptorPool(vulkanDevice, descriptorPoolInfo, null, this) }, {}) - DescriptorPool(handle, maxSets) + DescriptorPool(handle) } } @@ -422,8 +526,8 @@ open class VulkanDevice( ubo: VulkanUBO.UBODescriptor, type: Int = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER ): Long { - val pool = findAvailableDescriptorPool() - pool.free -= 1 + val pool = findAvailableDescriptorPool(type) + pool.decreaseAvailable(type, 1) logger.debug("Creating descriptor set with $bindingCount bindings, DSL=$descriptorSetLayout") return stackPush().use { stack -> @@ -432,7 +536,7 @@ open class VulkanDevice( val allocInfo = VkDescriptorSetAllocateInfo.calloc(stack) .sType(VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO) .pNext(MemoryUtil.NULL) - .descriptorPool(findAvailableDescriptorPool().handle) + .descriptorPool(pool.handle) .pSetLayouts(pDescriptorSetLayout) val descriptorSet = VU.getLong("createDescriptorSet", @@ -472,8 +576,8 @@ open class VulkanDevice( buffer: VulkanBuffer, size: Long = 2048L ): Long { - val pool = findAvailableDescriptorPool() - pool.free -= 1 + val pool = findAvailableDescriptorPool(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC) + pool.decreaseAvailable(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, 1) logger.debug("Creating dynamic descriptor set with $bindingCount bindings, DSL=${descriptorSetLayout.toHexString()}") @@ -483,7 +587,7 @@ open class VulkanDevice( val allocInfo = VkDescriptorSetAllocateInfo.calloc() .sType(VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO) .pNext(MemoryUtil.NULL) - .descriptorPool(findAvailableDescriptorPool().handle) + .descriptorPool(pool.handle) .pSetLayouts(pDescriptorSetLayout) val descriptorSet = VU.getLong("createDescriptorSet", @@ -634,8 +738,6 @@ open class VulkanDevice( imageLoadStore: Boolean = false, onlyFor: List? = null ): Long { - val pool = findAvailableDescriptorPool() - pool.free -= 1 return stackPush().use { stack -> val (type, layout) = if (!imageLoadStore) { @@ -644,12 +746,15 @@ open class VulkanDevice( VK_DESCRIPTOR_TYPE_STORAGE_IMAGE to VK_IMAGE_LAYOUT_GENERAL } + val pool = findAvailableDescriptorPool(type) + pool.decreaseAvailable(type) + val pDescriptorSetLayout = stack.callocLong(1).put(0, descriptorSetLayout) val allocInfo = VkDescriptorSetAllocateInfo.calloc(stack) .sType(VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO) .pNext(MemoryUtil.NULL) - .descriptorPool(findAvailableDescriptorPool().handle) + .descriptorPool(pool.handle) .pSetLayouts(pDescriptorSetLayout) val descriptorSet = VU.getLong("createDescriptorSet", @@ -697,12 +802,12 @@ open class VulkanDevice( * * Creates a new pool if necessary. */ - fun findAvailableDescriptorPool(requiredSets: Int = 1): DescriptorPool { - val available = descriptorPools.firstOrNull { it.free >= requiredSets } + fun findAvailableDescriptorPool(type: Int, requiredSets: Int = 1): DescriptorPool { + val available = descriptorPools.firstOrNull { it.free[type]!! >= requiredSets } return if(available == null) { descriptorPools.add(createDescriptorPool()) - descriptorPools.first { it.free >= requiredSets } + descriptorPools.first { it.free[type]!! >= requiredSets } } else { available } diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanFramebuffer.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanFramebuffer.kt index dd5a26eb75..4c1c61844e 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanFramebuffer.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanFramebuffer.kt @@ -702,7 +702,7 @@ open class VulkanFramebuffer(protected val device: VulkanDevice, /** * Helper function to set up Vulkan structs. */ - fun T.default(): T { + fun > T.default(): T { if(this is VkSamplerCreateInfo) { this.sType(VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO).pNext(NULL) } else if(this is VkFramebufferCreateInfo) { diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanImage.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanImage.kt new file mode 100644 index 0000000000..da189db146 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanImage.kt @@ -0,0 +1,376 @@ +package graphics.scenery.backends.vulkan + +import graphics.scenery.backends.vulkan.VulkanTexture.Companion.formatToString +import graphics.scenery.textures.UpdatableTexture +import graphics.scenery.utils.lazyLogger +import org.joml.Vector4i +import org.lwjgl.system.MemoryUtil +import org.lwjgl.vulkan.* + +/** + * Wrapper class for holding on to raw Vulkan [image]s backed by [memory]. + */ +class VulkanImage( + val device: VulkanDevice, + val width: Int, + val height: Int, + val depth: Int, + val format: Int, + val miplevels: Int, + var image: Long = -1L, + var memory: Long = -1L, + val maxSize: Long = -1L +) { + + /** Raw Vulkan sampler. */ + var sampler: Long = -1L + internal set + /** Raw Vulkan view. */ + var view: Long = -1L + internal set + + /** + * Copies the content of the image from [buffer]. This gets executed + * within a given [commandBuffer]. + */ + fun copyFrom(commandBuffer: VkCommandBuffer, buffer: VulkanBuffer, update: UpdatableTexture.TextureUpdate? = null, bufferOffset: Long = 0) { + with(commandBuffer) { + val bufferImageCopy = VkBufferImageCopy.calloc(1) + + bufferImageCopy.imageSubresource() + .aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT) + .mipLevel(0) + .baseArrayLayer(0) + .layerCount(1) + bufferImageCopy.bufferOffset(bufferOffset) + + if(update != null) { + logger.debug("Copying {} to {}", update.extents, buffer.vulkanBuffer.toHexString()) + bufferImageCopy.imageExtent().set(update.extents.w, update.extents.h, update.extents.d) + bufferImageCopy.imageOffset().set(update.extents.x, update.extents.y, update.extents.z) + } else { + logger.debug("Copying {}x{}x{} to {}", width, height, depth, buffer.vulkanBuffer.toHexString()) + bufferImageCopy.imageExtent().set(width, height, depth) + bufferImageCopy.imageOffset().set(0, 0, 0) + } + + VK10.vkCmdCopyBufferToImage( + this, + buffer.vulkanBuffer, + this@VulkanImage.image, VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + bufferImageCopy + ) + + bufferImageCopy.free() + } + + update?.let { + it.consumed = true + } + } + + /** + * Copies the content of the image to [buffer] from a series of [updates]. This gets executed + * within a given [commandBuffer]. + */ + fun copyFrom(commandBuffer: VkCommandBuffer, buffer: VulkanBuffer, updates: List, bufferOffset: Long = 0) { + logger.debug("Got {} texture updates for {}", updates.size, this) + with(commandBuffer) { + val bufferImageCopy = VkBufferImageCopy.calloc(1) + var offset = bufferOffset + + updates.forEach { update -> + val updateSize = update.contents.remaining() + bufferImageCopy.imageSubresource() + .aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT) + .mipLevel(0) + .baseArrayLayer(0) + .layerCount(1) + bufferImageCopy.bufferOffset(offset) + + bufferImageCopy.imageExtent().set(update.extents.w, update.extents.h, update.extents.d) + bufferImageCopy.imageOffset().set(update.extents.x, update.extents.y, update.extents.z) + + logger.debug("Copying {} to {}", update.extents, buffer.vulkanBuffer.toHexString()) + VK10.vkCmdCopyBufferToImage( + this, + buffer.vulkanBuffer, + this@VulkanImage.image, VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + bufferImageCopy + ) + + offset += updateSize + update.consumed = true + } + + bufferImageCopy.free() + } + } + + /** + * Copies the content of the image from a given [VulkanImage], [image]. + * This gets executed within a given [commandBuffer]. + */ + fun copyFrom(commandBuffer: VkCommandBuffer, image: VulkanImage, extents: UpdatableTexture.TextureExtents? = null) { + with(commandBuffer) { + val subresource = VkImageSubresourceLayers.calloc() + .aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT) + .baseArrayLayer(0) + .mipLevel(0) + .layerCount(1) + + val region = VkImageCopy.calloc(1) + .srcSubresource(subresource) + .dstSubresource(subresource) + + if(extents != null) { + region.srcOffset().set(extents.x, extents.y, extents.z) + region.dstOffset().set(extents.x, extents.y, extents.z) + region.extent().set(extents.w, extents.h, extents.d) + } else { + region.srcOffset().set(0, 0, 0) + region.dstOffset().set(0, 0, 0) + region.extent().set(width, height, depth) + } + + VK10.vkCmdCopyImage( + this, + image.image, VK10.VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + this@VulkanImage.image, VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + region + ) + + subresource.free() + region.free() + } + } + + /** + * Creates a [mipLevels] mipmaps of this [VulkanImage] based on the contents + * of the zero miplevel of the current image. All operations will be executed on + * a given [commandBuffer]. + */ + fun maybeCreateMipmaps(commandBuffer: VkCommandBuffer, mipLevels: Int) { + if(mipLevels == 1) { + return + } + + val imageBlit = VkImageBlit.calloc(1) + for (mipLevel in 1 until mipLevels) { + imageBlit.srcSubresource().set(VK10.VK_IMAGE_ASPECT_COLOR_BIT, mipLevel - 1, 0, 1) + imageBlit.srcOffsets(1).set(width shr (mipLevel - 1), height shr (mipLevel - 1), 1) + + val dstWidth = width shr mipLevel + val dstHeight = height shr mipLevel + + if (dstWidth < 2 || dstHeight < 2) { + break + } + + imageBlit.dstSubresource().set(VK10.VK_IMAGE_ASPECT_COLOR_BIT, mipLevel, 0, 1) + imageBlit.dstOffsets(1).set(width shr (mipLevel), height shr (mipLevel), 1) + + val mipSourceRange = VkImageSubresourceRange.calloc() + .aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT) + .baseArrayLayer(0) + .layerCount(1) + .baseMipLevel(mipLevel - 1) + .levelCount(1) + + val mipTargetRange = VkImageSubresourceRange.calloc() + .aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT) + .baseArrayLayer(0) + .layerCount(1) + .baseMipLevel(mipLevel) + .levelCount(1) + + if (mipLevel > 1) { + VulkanTexture.transitionLayout( + image, + VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + VK10.VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + subresourceRange = mipSourceRange, + srcStage = VK10.VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + dstStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT, + commandBuffer = commandBuffer + ) + } + + VulkanTexture.transitionLayout( + image, + VK10.VK_IMAGE_LAYOUT_UNDEFINED, + VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + subresourceRange = mipTargetRange, + srcStage = VK10.VK_PIPELINE_STAGE_HOST_BIT, + dstStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT, + commandBuffer = commandBuffer + ) + + VK10.vkCmdBlitImage( + commandBuffer, + image, + VK10.VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + image, + VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + imageBlit, + VK10.VK_FILTER_LINEAR + ) + + VulkanTexture.transitionLayout( + image, + VK10.VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + subresourceRange = mipSourceRange, + srcStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT, + dstStage = VK10.VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + commandBuffer = commandBuffer + ) + + VulkanTexture.transitionLayout( + image, + VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + subresourceRange = mipTargetRange, + srcStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT, + dstStage = VK10.VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, + commandBuffer = commandBuffer + ) + + mipSourceRange.free() + mipTargetRange.free() + } + + imageBlit.free() + } + + /** + * Creates a Vulkan image view containing [miplevels] miplevels for this image, with components swizzled + * according to [swizzle], which is null by default. Will erase the default view, if it exists. + */ + fun createView(miplevels: Int = this.miplevels, swizzle: Vector4i? = null): Long { + if(view != -1L) { + VK10.vkDestroyImageView(device.vulkanDevice, view, null) + } + + val subresourceRange = VkImageSubresourceRange.calloc().set(VK10.VK_IMAGE_ASPECT_COLOR_BIT, 0, miplevels, 0, 1) + + val vi = VkImageViewCreateInfo.calloc() + .sType(VK10.VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO) + .pNext(MemoryUtil.NULL) + .image(image) + .viewType(if (depth > 1) { + VK10.VK_IMAGE_VIEW_TYPE_3D + } else { + VK10.VK_IMAGE_VIEW_TYPE_2D + }) + .format(format) + .subresourceRange(subresourceRange) + + swizzle?.let {s -> + vi.components().set( + s.x, + s.y, + s.z, + s.w + ) + } + + return VU.getLong("Creating image view", + { VK10.vkCreateImageView(device.vulkanDevice, vi, null, this) }, + { vi.free(); subresourceRange.free(); }) + } + + /** + * Returns a string representation of this [VulkanImage]. + */ + override fun toString(): String { + return "VulkanImage (${this.image.toHexString()}, ${width}x${height}x${depth}, format=${format.formatToString()}, maxSize=${this.maxSize})" + } + + companion object { + private val logger by lazyLogger() + + /** + * Creates a [VulkanImage] on [device], of [format] with a given [width], [height], and [depth]. + * [usage] and [memoryFlags] need to be given, as well as the [tiling] parameter and number of [mipLevels]. + * A custom memory allocator may be used and given as [customAllocator]. + */ + fun create( + device: VulkanDevice, + width: Int, + height: Int, + depth: Int, + format: Int, + usage: Int, + tiling: Int, + memoryFlags: Int, + mipLevels: Int, + initialLayout: Int? = null, + customAllocator: ((VkMemoryRequirements, Long) -> Long)? = null, + imageCreateInfo: VkImageCreateInfo? = null + ): VulkanImage { + val imageInfo = if(imageCreateInfo != null) { + imageCreateInfo + } else { + val i = VkImageCreateInfo.calloc() + .sType(VK10.VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO) + .imageType( + if(depth == 1) { + VK10.VK_IMAGE_TYPE_2D + } else { + VK10.VK_IMAGE_TYPE_3D + } + ) + .mipLevels(mipLevels) + .arrayLayers(1) + .format(format) + .tiling(tiling) + .initialLayout( + if(depth == 1) { + VK10.VK_IMAGE_LAYOUT_PREINITIALIZED + } else { + VK10.VK_IMAGE_LAYOUT_UNDEFINED + } + ) + .usage(usage) + .sharingMode(VK10.VK_SHARING_MODE_EXCLUSIVE) + .samples(VK10.VK_SAMPLE_COUNT_1_BIT) + .flags(VK10.VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) + + i.extent().set(width, height, depth) + i + } + + if(initialLayout != null) { + imageInfo.initialLayout(initialLayout) + } + + val image = VU.getLong("create staging image", + { VK10.vkCreateImage(device.vulkanDevice, imageInfo, null, this) }, {}) + + val reqs = VkMemoryRequirements.calloc() + VK10.vkGetImageMemoryRequirements(device.vulkanDevice, image, reqs) + val memorySize = reqs.size() + + val memory = if(customAllocator == null) { + val allocInfo = VkMemoryAllocateInfo.calloc() + .sType(VK10.VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO) + .pNext(MemoryUtil.NULL) + .allocationSize(memorySize) + .memoryTypeIndex(device.getMemoryType(reqs.memoryTypeBits(), memoryFlags).first()) + + VU.getLong("allocate image staging memory of size $memorySize", + { VK10.vkAllocateMemory(device.vulkanDevice, allocInfo, null, this) }, + { imageInfo.free(); allocInfo.free() }) + } else { + customAllocator.invoke(reqs, image) + } + + reqs.free() + + VK10.vkBindImageMemory(device.vulkanDevice, image, memory, 0) + + return VulkanImage(device, width, height, depth, format, mipLevels, image, memory, memorySize) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanNodeHelpers.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanNodeHelpers.kt index 20aaa464f5..0d9273fc6f 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanNodeHelpers.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanNodeHelpers.kt @@ -4,7 +4,6 @@ import graphics.scenery.* import graphics.scenery.backends.* import graphics.scenery.attribute.renderable.Renderable import graphics.scenery.attribute.material.Material -import graphics.scenery.backends.vulkan.VulkanTexture.Companion.formatToString import graphics.scenery.backends.vulkan.VulkanTexture.Companion.toVulkanFormat import graphics.scenery.textures.Texture import graphics.scenery.textures.UpdatableTexture @@ -12,7 +11,6 @@ import graphics.scenery.utils.lazyLogger import org.lwjgl.system.jemalloc.JEmalloc import org.lwjgl.vulkan.VK10 import org.lwjgl.vulkan.VkBufferCopy -import org.lwjgl.vulkan.VkQueue import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.concurrent.atomic.AtomicInteger @@ -44,7 +42,7 @@ object VulkanNodeHelpers { stagingPool: VulkanBufferPool, geometryPool: VulkanBufferPool, commandPools: VulkanRenderer.CommandPools, - queue: VkQueue + queue: VulkanDevice.QueueWithMutex ): VulkanObjectState { val geometry = node.geometryOrNull() ?: return state val vertices = geometry.vertices.duplicate() @@ -160,7 +158,7 @@ object VulkanNodeHelpers { * Updates instance buffers for a given [node] on [device]. Modifies the [node]'s [state] * and allocates necessary command buffers from [commandPools] and submits to [queue]. Returns the [node]'s modified [VulkanObjectState]. */ - fun updateInstanceBuffer(device: VulkanDevice, node: InstancedNode, state: VulkanObjectState, commandPools: VulkanRenderer.CommandPools, queue: VkQueue): VulkanObjectState { + fun updateInstanceBuffer(device: VulkanDevice, node: InstancedNode, state: VulkanObjectState, commandPools: VulkanRenderer.CommandPools, queue: VulkanDevice.QueueWithMutex): VulkanObjectState { logger.trace("Updating instance buffer for {}", node.name) // parentNode.instances is a CopyOnWrite array list, and here we keep a reference to the original. @@ -263,7 +261,7 @@ object VulkanNodeHelpers { * * Returns a [Pair] of [Boolean], indicating whether contents or descriptor set have changed. */ - fun loadTexturesForNode(device: VulkanDevice, node: Node, s: VulkanObjectState, defaultTextures: Map, textureCache: MutableMap, commandPools: VulkanRenderer.CommandPools, queue: VkQueue): Pair { + fun loadTexturesForNode(device: VulkanDevice, node: Node, s: VulkanObjectState, defaultTextures: Map, textureCache: MutableMap, commandPools: VulkanRenderer.CommandPools, queue: VulkanDevice.QueueWithMutex, transferQueue: VulkanDevice.QueueWithMutex = queue): Pair { val material = node.materialOrNull() ?: return Pair(false, false) val defaultTexture = defaultTextures["DefaultTexture"] ?: throw IllegalStateException("Default fallback texture does not exist.") // if a node is not yet initialized, we'll definitely require a new DS @@ -296,13 +294,15 @@ object VulkanNodeHelpers { val t: VulkanTexture = if (existingTexture != null && existingTexture.canBeReused(texture, miplevels, device) && existingTexture != defaultTexture) { + existingTexture.texture = texture existingTexture } else { descriptorUpdated = true - VulkanTexture(device, commandPools, queue, queue, texture, miplevels) + VulkanTexture(device, commandPools, queue, transferQueue, texture, miplevels) } texture.contents?.let { contents -> + logger.debug("Copying contents of size ${contents.remaining()/1024/1024}M") t.copyFrom(contents.duplicate()) } @@ -412,6 +412,8 @@ object VulkanNodeHelpers { val pipeline = pass.initializePipeline(shaderModules, material.cullingMode, material.depthTest, + material.depthWrite, + material.depthOp, material.blending, material.wireframe, s.vertexDescription diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanObjectState.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanObjectState.kt index c6a8231a2f..24908a09b2 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanObjectState.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanObjectState.kt @@ -189,8 +189,8 @@ open class VulkanObjectState { val pDescriptorSetLayout = memAllocLong(1) pDescriptorSetLayout.put(0, descriptorSetLayout) - val pool = device.findAvailableDescriptorPool() - pool.free -= 1 + val pool = device.findAvailableDescriptorPool(VK10.VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) + pool.decreaseAvailable(VK10.VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) val allocInfo = VkDescriptorSetAllocateInfo.calloc() .sType(VK10.VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO) diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt index 518bd69479..9b17c20a36 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt @@ -8,6 +8,7 @@ import graphics.scenery.attribute.renderable.Renderable import graphics.scenery.backends.* import graphics.scenery.textures.Texture import graphics.scenery.utils.* +import graphics.scenery.utils.extensions.applyVulkanCoordinateSystem import kotlinx.coroutines.* import org.joml.* import org.lwjgl.PointerBuffer @@ -28,7 +29,6 @@ import org.lwjgl.vulkan.KHRWin32Surface.VK_KHR_WIN32_SURFACE_EXTENSION_NAME import org.lwjgl.vulkan.KHRXlibSurface.VK_KHR_XLIB_SURFACE_EXTENSION_NAME import org.lwjgl.vulkan.MVKMacosSurface.VK_MVK_MACOS_SURFACE_EXTENSION_NAME import org.lwjgl.vulkan.VK10.* -import java.awt.BorderLayout import java.awt.image.BufferedImage import java.awt.image.DataBufferByte import java.io.File @@ -37,9 +37,10 @@ import java.nio.IntBuffer import java.nio.LongBuffer import java.util.* import java.util.concurrent.* +import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.ReentrantLock import javax.imageio.ImageIO -import javax.swing.JFrame +import kotlin.collections.ArrayList import kotlin.concurrent.thread import kotlin.concurrent.withLock import kotlin.reflect.full.* @@ -152,7 +153,7 @@ open class VulkanRenderer(hub: Hub, if(lock.tryLock() && !shouldClose) { logger.info("Recreating Swapchain at frame $frames (${swapchain.javaClass.simpleName})") // create new swapchain with changed surface parameters - vkQueueWaitIdle(queue) + vkQueueWaitIdle(queue.queue) with(VU.newCommandBuffer(device, commandPools.Standard, autostart = true)) { // Create the swapchain (this will also add a memory barrier to initialize the framebuffer images) @@ -216,28 +217,38 @@ open class VulkanRenderer(hub: Hub, } } + private fun Throwable.filteredStackTrace(): List { + val firstIndex = this.stackTrace.indexOfFirst { it.toString().contains("VkDebugUtilsMessengerCallbackEXTI") } + 1 + val lastIndex = this.stackTrace.indexOfLast { it.toString().contains("SceneryBase") } - 1 + + return this.stackTrace.copyOfRange(firstIndex, lastIndex) + .filter { !it.toString().contains(".coroutines.") } + .map { " at $it" } + } + var debugCallbackUtils = callback@ { severity: Int, type: Int, callbackDataPointer: Long, _: Long -> val dbg = if (type and VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT == VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT) { " (performance)" } else if(type and VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT == VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT) { - " (validation)" + "" } else { "" } val callbackData = VkDebugUtilsMessengerCallbackDataEXT.create(callbackDataPointer) - val obj = callbackData.pMessageIdNameString() val message = callbackData.pMessageString() val objectType = 0 when (severity) { VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT -> - logger.error("!! $obj($objectType) Validation$dbg: $message") + logger.error("🌋⛔️ Validation$dbg: $message") VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT -> - logger.warn("!! $obj($objectType) Validation$dbg: $message") + logger.warn("🌋⚠️ Validation$dbg: $message") VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT -> - logger.info("!! $obj($objectType) Validation$dbg: $message") - else -> logger.info("!! $obj($objectType) Validation (unknown message type)$dbg: $message") + logger.info("🌋ℹ️ Validation$dbg: $message") + VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT -> + logger.info("(verbose) $message") + else -> logger.info("🌋🤷 Validation (unknown message type)$dbg: $message") } // trigger exception and delay if strictValidation is activated in general, or only for specific object types @@ -249,12 +260,8 @@ open class VulkanRenderer(hub: Hub, // set 15s of delay until the next frame is rendered if a validation error happens renderDelay = System.getProperty("scenery.VulkanRenderer.DefaultRenderDelay", "1500").toLong() - try { - throw VulkanValidationLayerException("Vulkan validation layer exception, see validation layer error messages above. To disable these exceptions, set scenery.VulkanRenderer.StrictValidation=false. Stack trace:") - } catch (e: VulkanValidationLayerException) { - logger.error(e.message) - e.printStackTrace() - } + val e = VulkanValidationLayerException("Vulkan validation layer exception, see validation layer error messages above. To disable these exceptions, set scenery.VulkanRenderer.StrictValidation=false.") + e.filteredStackTrace().forEach { logger.error(it) } } // return false here, otherwise the application would quit upon encountering a validation error. @@ -292,12 +299,8 @@ open class VulkanRenderer(hub: Hub, // set 15s of delay until the next frame is rendered if a validation error happens renderDelay = System.getProperty("scenery.VulkanRenderer.DefaultRenderDelay", "1500").toLong() - try { - throw VulkanValidationLayerException("Vulkan validation layer exception, see validation layer error messages above. To disable these exceptions, set scenery.VulkanRenderer.StrictValidation=false. Stack trace:") - } catch (e: VulkanValidationLayerException) { - logger.error(e.message) - e.printStackTrace() - } + val e = VulkanValidationLayerException("Vulkan validation layer exception, see validation layer error messages above. To disable these exceptions, set scenery.VulkanRenderer.StrictValidation=false. Stack trace:") + e.filteredStackTrace().forEach { logger.error(it) } } // return false here, otherwise the application would quit upon encountering a validation error. @@ -330,6 +333,8 @@ open class VulkanRenderer(hub: Hub, private var recordMovieOverwrite: Boolean = false override var pushMode: Boolean = false + val persistentTextureRequests = ArrayList>() + var scene: Scene = Scene() protected var sceneArray: HashSet = HashSet(256) @@ -347,8 +352,8 @@ open class VulkanRenderer(hub: Hub, protected var debugCallbackHandle: Long = -1L // Create static Vulkan resources - protected var queue: VkQueue - protected var transferQueue: VkQueue + protected var queue: VulkanDevice.QueueWithMutex + protected var transferQueue: VulkanDevice.QueueWithMutex protected var swapchain: Swapchain protected var ph = PresentHelpers() @@ -379,7 +384,6 @@ open class VulkanRenderer(hub: Hub, var fps = 0 protected set protected var frames = 0 - protected var totalFrames = 0L protected var renderDelay = 0L protected var heartbeatTimer = Timer() protected var gpuStats: GPUStats? = null @@ -387,13 +391,6 @@ open class VulkanRenderer(hub: Hub, private var renderConfig: RenderConfigReader.RenderConfig private var flow: List = listOf() - private val vulkanProjectionFix = - Matrix4f( - 1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, -1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.5f, 0.0f, - 0.0f, 0.0f, 0.5f, 1.0f) - final override var renderConfigFile: String = "" set(config) { field = config @@ -514,7 +511,9 @@ open class VulkanRenderer(hub: Hub, stack.UTF8(EXTMetalSurface.VK_EXT_METAL_SURFACE_EXTENSION_NAME) ) } else { - null + stack.pointers( + stack.UTF8(VK_KHR_SURFACE_EXTENSION_NAME) + ) } createInstance( @@ -576,8 +575,15 @@ open class VulkanRenderer(hub: Hub, val headless = (selectedSwapchain?.kotlin?.companionObjectInstance as? SwapchainParameters)?.headless ?: false device = VulkanDevice.fromPhysicalDevice(instance, - physicalDeviceFilter = { _, device -> "${device.vendor} ${device.name}".contains(System.getProperty("scenery.Renderer.Device", "DOES_NOT_EXIST"))}, - additionalExtensions = { physicalDevice -> hub.getWorkingHMDDisplay()?.getVulkanDeviceExtensions(physicalDevice)?.toMutableList() ?: mutableListOf() }, + physicalDeviceFilter = { index, device -> + val namePreference = System.getProperty("scenery.Renderer.Device", "DOES_NOT_EXIST") + val idPreference = System.getProperty("scenery.Renderer.DeviceId", "DOES_NOT_EXIST").toIntOrNull() + when { + idPreference != null -> index == idPreference + else -> "${device.vendor} ${device.name}".contains(namePreference) + } + }, + additionalExtensions = { physicalDevice -> hub.getWorkingHMDDisplay()?.getVulkanDeviceExtensions(physicalDevice)?.toMutableList() ?: mutableListOf(KHRSwapchain.VK_KHR_SWAPCHAIN_EXTENSION_NAME) }, validationLayers = requestedValidationLayers, headless = headless, debugEnabled = validation @@ -597,15 +603,16 @@ open class VulkanRenderer(hub: Hub, } } - queue = VU.createDeviceQueue(device, device.queues.graphicsQueue.first) - logger.debug("Creating transfer queue with ${device.queues.transferQueue.first} (vs ${device.queues.graphicsQueue})") - transferQueue = VU.createDeviceQueue(device, device.queues.transferQueue.first) + queue = device.getQueue(device.queueIndices.graphicsQueue.first) + logger.debug("Creating transfer queue with ${device.queueIndices.transferQueue.first} (vs ${device.queueIndices.graphicsQueue.first})") + transferQueue = device.getQueue(device.queueIndices.transferQueue.first) + logger.debug("Have {} and {}", queue, transferQueue) with(commandPools) { - Render = device.createCommandPool(device.queues.graphicsQueue.first) - Standard = device.createCommandPool(device.queues.graphicsQueue.first) - Compute = device.createCommandPool(device.queues.computeQueue.first) - Transfer = device.createCommandPool(device.queues.transferQueue.first) + Render = device.createCommandPool(device.queueIndices.graphicsQueue.first) + Standard = device.createCommandPool(device.queueIndices.graphicsQueue.first) + Compute = device.createCommandPool(device.queueIndices.computeQueue.first) + Transfer = device.createCommandPool(device.queueIndices.transferQueue.first) } logger.debug("Creating command pools done") @@ -613,13 +620,14 @@ open class VulkanRenderer(hub: Hub, swapchain = when { selectedSwapchain != null -> { logger.info("Using swapchain ${selectedSwapchain.simpleName}") - val params = selectedSwapchain.kotlin.primaryConstructor!!.parameters.associate { param -> - param to when(param.name) { + val params = selectedSwapchain.kotlin.primaryConstructor!!.parameters.associateWith { param -> + when(param.name) { "device" -> device "queue" -> queue "commandPools" -> commandPools "renderConfig" -> renderConfig "useSRGB" -> renderConfig.sRGB + "vsync" -> !settings.get("Renderer.DisableVsync") else -> null } }.filter { it.value != null } @@ -690,13 +698,13 @@ open class VulkanRenderer(hub: Hub, } val validationsEnabled = if (validation) { - " - VALIDATIONS ENABLED" + ", validation layers enabled" } else { "" } - if(embedIn == null) { - window.title = "$applicationName [${this@VulkanRenderer.javaClass.simpleName}, ${this@VulkanRenderer.renderConfig.name}] $validationsEnabled - $fps fps" + if(embedIn == null || (embedIn as? SceneryJPanel)?.owned == true) { + window.title = "$applicationName [${this@VulkanRenderer.renderConfig.name}$validationsEnabled] $fps fps" } } }, 0, 1000) @@ -850,7 +858,10 @@ open class VulkanRenderer(hub: Hub, return false } - if (s.initialized) return true + if (s.initialized) { + node.initialized = true + return true + } s.flags.add(RendererFlags.Seen) @@ -921,7 +932,16 @@ open class VulkanRenderer(hub: Hub, } } - val (_, descriptorUpdated) = VulkanNodeHelpers.loadTexturesForNode(device, node, s, defaultTextures, textureCache, commandPools, queue) + val (_, descriptorUpdated) = VulkanNodeHelpers.loadTexturesForNode( + device, + node, + s, + defaultTextures, + textureCache, + commandPools, + queue, + transferQueue + ) if(descriptorUpdated) { s.texturesToDescriptorSets(device, renderpasses.filter { it.value.passConfig.type != RenderConfigReader.RenderpassType.quad }, @@ -933,13 +953,14 @@ open class VulkanRenderer(hub: Hub, val materialUbo = VulkanUBO(device, backingBuffer = buffers.UBOs) with(materialUbo) { name = "MaterialProperties" - add("materialType", { node.materialOrNull()!!.materialTypeFromTextures(s) }) + add("materialType", { node.materialOrNull()!!.materialTypeFromTextures(renderable.rendererMetadata()!!) }) add("Ka", { node.materialOrNull()!!.ambient }) add("Kd", { node.materialOrNull()!!.diffuse }) add("Ks", { node.materialOrNull()!!.specular }) add("Roughness", { node.materialOrNull()!!.roughness}) add("Metallic", { node.materialOrNull()!!.metallic}) add("Opacity", { node.materialOrNull()!!.blending.opacity }) + add("Emissive", { node.materialOrNull()!!.emissive }) createUniformBuffer() s.UBOs.put("MaterialProperties", materialPropertiesDescriptorSet.contents to this) @@ -1222,7 +1243,7 @@ open class VulkanRenderer(hub: Hub, } protected fun prepareDefaultTextures(device: VulkanDevice) { - val t = VulkanTexture.loadFromFile(device, commandPools, queue, queue, + val t = VulkanTexture.loadFromFile(device, commandPools, queue, transferQueue, Renderer::class.java.getResourceAsStream("DefaultTexture.png"), "png", true, true) // TODO: Do an asset manager or sth here? @@ -1288,7 +1309,7 @@ open class VulkanRenderer(hub: Hub, } private suspend fun submitFrame( - queue: VkQueue, + queue: VulkanDevice.QueueWithMutex, pass: VulkanRenderpass, commandBuffer: VulkanCommandBuffer, present: PresentHelpers @@ -1310,9 +1331,7 @@ open class VulkanRenderer(hub: Hub, val q = (swapchain as? VulkanSwapchain)?.presentQueue ?: queue // Submit to the graphics queue // vkResetFences(device.vulkanDevice, swapchain.currentFence) - VU.run("Submit viewport render queue", { - vkQueueSubmit(q, present.submitInfo, swapchain.currentFence) - }) + VU.run("Submit viewport render queue", { vkQueueSubmit(q.queue, present.submitInfo, swapchain.currentFence) }) // submit to OpenVR if attached if(hub?.getWorkingHMDDisplay()?.hasCompositor() == true) { @@ -1346,7 +1365,7 @@ open class VulkanRenderer(hub: Hub, val ref = VulkanTexture.getReference(req.first) if(ref != null) { - ref.copyTo(buffer) + ref.copyTo(buffer, false) req.second.send(req.first) req.second.close() logger.info("Sent updated texture") @@ -1356,6 +1375,21 @@ open class VulkanRenderer(hub: Hub, } } + persistentTextureRequests.forEach { (texture, indicator) -> + val ref = VulkanTexture.getReference(texture) + val buffer = texture.contents ?: return@forEach + + if(ref != null) { + val start = System.nanoTime() + ref.copyTo(buffer) + val end = System.nanoTime() + logger.debug("The request textures of size ${texture.contents?.remaining()?.toFloat()?.div((1024f*1024f))} took: ${(end.toDouble()-start.toDouble())/1000000.0}") + indicator.incrementAndGet() + } else { + logger.error("In persistent texture requests: Texture not accessible") + } + } + if (recordMovie || screenshotRequested || imageRequests.isNotEmpty()) { val request = try { imageRequests.poll() @@ -1517,6 +1551,10 @@ open class VulkanRenderer(hub: Hub, hub?.getWorkingHMDDisplay()?.wantsVR(settings)?.update() } + runAfterRendering.forEach { + it.invoke() + } + val presentDuration = System.nanoTime() - startPresent stats?.add("Renderer.viewportSubmitAndPresent", presentDuration) @@ -1527,6 +1565,21 @@ open class VulkanRenderer(hub: Hub, private var previousFrame = 0 private var currentNow = 0L + /** + * Enum class for reasons to re-record command buffers. + */ + enum class RerecordingCause { + GeometryUpdated, + TexturesUpdated, + MaterialUpdated, + InstancesUpdated, + SceneUpdated, + SwapchainUpdated + } + + /** Keeps track of the reasons why push mode has been defeated, and in which frame. */ + val pushModeDefeats: Queue>>> = LinkedList() + /** * This function renders the scene */ @@ -1585,8 +1638,13 @@ open class VulkanRenderer(hub: Hub, // flag set to true if command buffer re-recording is necessary, // e.g. because of scene or pipeline changes - var forceRerecording = instancesUpdated - val rerecordingCauses = ArrayList(20) + var forceRerecording = false + val rerecordingCauses = ArrayList>(20) + + if(instancesUpdated) { + forceRerecording = true + rerecordingCauses.add("General" to RerecordingCause.InstancesUpdated) + } profiler?.begin("Renderer.PreDraw") // here we discover the objects in the scene that could be relevant for the scene @@ -1626,7 +1684,7 @@ open class VulkanRenderer(hub: Hub, updateNodeGeometry(node) dirty = false - rerecordingCauses.add(node.name) + rerecordingCauses.add(node.name to RerecordingCause.GeometryUpdated) forceRerecording = true } } @@ -1641,14 +1699,23 @@ open class VulkanRenderer(hub: Hub, } val reloadTime = measureTimeMillis { - val (texturesUpdatedForNode, descriptorUpdated) = VulkanNodeHelpers.loadTexturesForNode(device, node, metadata, defaultTextures, textureCache, commandPools, queue) + val (texturesUpdatedForNode, descriptorUpdated) = VulkanNodeHelpers.loadTexturesForNode( + device, + node, + metadata, + defaultTextures, + textureCache, + commandPools, + queue, + transferQueue + ) if(descriptorUpdated) { metadata.texturesToDescriptorSets(device, renderpasses.filter { it.value.passConfig.type != RenderConfigReader.RenderpassType.quad }, renderable) logger.trace("Force command buffer re-recording, as reloading textures for ${node.name}") - rerecordingCauses.add(node.name) + rerecordingCauses.add(node.name to RerecordingCause.TexturesUpdated) forceRerecording = true } @@ -1671,7 +1738,7 @@ open class VulkanRenderer(hub: Hub, renderable) } - rerecordingCauses.add(node.name) + rerecordingCauses.add(node.name to RerecordingCause.MaterialUpdated) forceRerecording = true (material as? ShaderMaterial)?.shaders?.stale = false @@ -1683,6 +1750,7 @@ open class VulkanRenderer(hub: Hub, val newSceneArray = sceneNodes.toHashSet() if (!newSceneArray.equals(sceneArray)) { forceRerecording = true + rerecordingCauses.add("General" to RerecordingCause.SceneUpdated) } sceneArray = newSceneArray @@ -1707,6 +1775,17 @@ open class VulkanRenderer(hub: Hub, swapchainRecreator.mustRecreate = true } + if(swapchainChanged && pushMode) { + rerecordingCauses.add("General" to RerecordingCause.SwapchainUpdated) + } + + if(pushMode && rerecordingCauses.size > 0) { + if(pushModeDefeats.size > 5) { + pushModeDefeats.poll() + } + pushModeDefeats.add(frames to rerecordingCauses) + } + profiler?.begin("Renderer.BeginFrame") val presentedFrames = swapchain.presentedFrames() // return if neither UBOs were updated, nor the scene was modified @@ -1777,7 +1856,7 @@ open class VulkanRenderer(hub: Hub, } // logger.info("Submitting pass $t waiting on semaphore ${target.waitSemaphores.get(0).toHexString()}") - VU.run("Submit pass $t render queue", { vkQueueSubmit(queue, si, commandBuffer.getFence() )}) + VU.run("Submit pass $t render queue", { vkQueueSubmit(queue.queue, si, commandBuffer.getFence() )}) commandBuffer.submitted = true waitSemaphore = targetSemaphore @@ -2061,13 +2140,6 @@ open class VulkanRenderer(hub: Hub, instanceMasters.isNotEmpty() } - fun Matrix4f.applyVulkanCoordinateSystem(): Matrix4f { - val m = Matrix4f(vulkanProjectionFix) - m.mul(this) - - return m - } - private fun getDescriptorCache(): TimestampedConcurrentHashMap> { @Suppress("UNCHECKED_CAST") @@ -2255,7 +2327,7 @@ open class VulkanRenderer(hub: Hub, initialized = false logger.info("Renderer teardown started.") - vkQueueWaitIdle(queue) + vkQueueWaitIdle(queue.queue) logger.debug("Closing nodes...") scene.discover(scene, { true }).forEach { @@ -2375,7 +2447,7 @@ open class VulkanRenderer(hub: Hub, fun setConfigSetting(key: String, value: Any) { val setting = "Renderer.$key" - logger.debug("Setting $setting: ${settings.get(setting)} -> $value") + logger.debug("Setting {}: {} -> {}", setting, settings.getOrNull(setting), value) settings.set(setting, value) } diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt index 8420d14701..4553f5e5c8 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt @@ -434,7 +434,9 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren private data class PipelineConfig( val shaderModules: HashSet, val cullingMode: Material.CullingMode, - val depthTest: Material.DepthTest, + val depthTest: Boolean, + val depthWrite: Boolean, + val depthOp: Material.DepthTest, val blending: Blending, val wireframe: Boolean, val vertexDescription: VulkanRenderer.VertexDescription @@ -450,7 +452,7 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren Material.CullingMode.FrontAndBack -> rasterizationState.cullMode(VK_CULL_MODE_FRONT_AND_BACK) } - when(config.depthTest) { + when(config.depthOp) { Material.DepthTest.Equal -> depthStencilState.depthCompareOp(VK_COMPARE_OP_EQUAL) Material.DepthTest.Less -> depthStencilState.depthCompareOp(VK_COMPARE_OP_LESS) Material.DepthTest.Greater -> depthStencilState.depthCompareOp(VK_COMPARE_OP_GREATER) @@ -460,6 +462,9 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren Material.DepthTest.Never -> depthStencilState.depthCompareOp(VK_COMPARE_OP_NEVER) } + depthStencilState.depthTestEnable(config.depthTest) + depthStencilState.depthWriteEnable(config.depthWrite) + if(config.wireframe) { rasterizationState.polygonMode(VK_POLYGON_MODE_LINE) } else { @@ -494,10 +499,12 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren pipelines["preferred-${renderable.getUuid()}"] = pipeline } - fun initializePipeline(shaderModules: List, cullingMode: Material.CullingMode, depthTest: Material.DepthTest, blending: Blending, wireframe: Boolean, vertexDescription: VulkanRenderer.VertexDescription?): VulkanPipeline { + fun initializePipeline(shaderModules: List, cullingMode: Material.CullingMode, depthTest: Boolean, depthWrite: Boolean, depthOp: Material.DepthTest, blending: Blending, wireframe: Boolean, vertexDescription: VulkanRenderer.VertexDescription?): VulkanPipeline { val config = PipelineConfig(hashSetOf(*shaderModules.toTypedArray()), cullingMode, depthTest, + depthWrite, + depthOp, blending, wireframe, vertexDescription ?: vertexDescriptors.getValue(VulkanRenderer.VertexDataKinds.PositionNormalTexcoord)) @@ -904,7 +911,7 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren config: RenderConfigReader.RenderConfig, device: VulkanDevice, commandPools: VulkanRenderer.CommandPools, - queue: VkQueue, + queue: VulkanDevice.QueueWithMutex, vertexDescriptors: ConcurrentHashMap, swapchain: Swapchain, windowWidth: Int, diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanScenePass.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanScenePass.kt index 772ca9c0d8..254203dd0a 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanScenePass.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanScenePass.kt @@ -20,6 +20,7 @@ import org.lwjgl.system.MemoryStack import org.lwjgl.system.MemoryUtil import org.lwjgl.vulkan.* import java.util.* +import kotlin.math.ceil /** * Helper object for scene pass command buffer recording. @@ -224,9 +225,9 @@ object VulkanScenePass { commandBuffer.device.deviceData.properties.limits().maxComputeWorkGroupCount().get(maxGroupCount) val groupCount = intArrayOf( - metadata.workSizes.x()/localSizes.first, - metadata.workSizes.y()/localSizes.second, - metadata.workSizes.z()/localSizes.third) + ceil(metadata.workSizes.x().toFloat()/localSizes.first.toFloat()).toInt(), + ceil(metadata.workSizes.y().toFloat()/localSizes.second.toFloat()).toInt(), + ceil(metadata.workSizes.z().toFloat()/localSizes.third.toFloat()).toInt()) groupCount.forEachIndexed { i, gc -> if(gc > maxGroupCount[i]) { @@ -599,7 +600,7 @@ object VulkanScenePass { } if(ds is VulkanRenderer.DescriptorSet.DynamicSet && ds.offset == BUFFER_OFFSET_UNINTIALISED ) { - logger.info("${node.name} has uninitialised UBO offset, skipping for rendering") + logger.debug("${node.name} has uninitialised UBO offset, skipping for rendering") skip = true } diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanShaderModule.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanShaderModule.kt index 02f6bfb2f7..284294dffe 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanShaderModule.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanShaderModule.kt @@ -224,8 +224,12 @@ open class VulkanShaderModule(val device: VulkanDevice, entryPoint: String, val fun getFromCacheOrCreate(device: VulkanDevice, entryPoint: String, sp: ShaderPackage): VulkanShaderModule { val signature = ShaderSignature(device, sp).hashCode() - return shaderModuleCache.getOrPut(signature) { + return if(sp.disableCaching) { VulkanShaderModule(device, entryPoint, sp) + } else { + shaderModuleCache.getOrPut(signature) { + VulkanShaderModule(device, entryPoint, sp) + } } } diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanSwapchain.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanSwapchain.kt index 7ea2c342f7..9ab6fd511f 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanSwapchain.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanSwapchain.kt @@ -30,7 +30,7 @@ import java.util.* * @author Ulrik Günther */ open class VulkanSwapchain(open val device: VulkanDevice, - open val queue: VkQueue, + open val queue: VulkanDevice.QueueWithMutex, open val commandPools: VulkanRenderer.CommandPools, @Suppress("unused") open val renderConfig: RenderConfigReader.RenderConfig, open val useSRGB: Boolean = true, @@ -57,7 +57,7 @@ open class VulkanSwapchain(open val device: VulkanDevice, /** Present info, allocated only once and reused. */ var presentInfo: VkPresentInfoKHR = VkPresentInfoKHR.calloc() /** Vulkan queue used exclusively for presentation. */ - lateinit var presentQueue: VkQueue + lateinit var presentQueue: VulkanDevice.QueueWithMutex /** Surface of the window to render into. */ open var surface: Long = 0 @@ -405,7 +405,7 @@ open class VulkanSwapchain(open val device: VulkanDevice, } logger.info("Present queue is ${presentQueueNodeIndex}, graphics queue is ${graphicsQueueNodeIndex}") - presentQueue = VU.createDeviceQueue(device, device.queues.graphicsQueue.first) + presentQueue = device.getQueue(device.queueIndices.graphicsQueue.first) // Get list of supported formats val formatCount = VU.getInts("Getting supported surface formats", 1, @@ -457,7 +457,7 @@ open class VulkanSwapchain(open val device: VulkanDevice, // here we accept the VK_ERROR_OUT_OF_DATE_KHR error code, which // seems to spuriously occur on Linux upon resizing. VU.run("Presenting swapchain image", - { KHRSwapchain.vkQueuePresentKHR(presentQueue, presentInfo) }, + { KHRSwapchain.vkQueuePresentKHR(presentQueue.queue, presentInfo) }, allowedResults = listOf(VK_ERROR_OUT_OF_DATE_KHR)) presentedFrames++ @@ -596,8 +596,8 @@ open class VulkanSwapchain(open val device: VulkanDevice, * Closes the swapchain, deallocating all of its resources. */ override fun close() { - vkQueueWaitIdle(presentQueue) - vkQueueWaitIdle(queue) + vkQueueWaitIdle(presentQueue.queue) + vkQueueWaitIdle(queue.queue) logger.debug("Closing swapchain $this") KHRSwapchain.vkDestroySwapchainKHR(device.vulkanDevice, handle, null) diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanTexture.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanTexture.kt index 9966fe5b2a..4b480f2071 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanTexture.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanTexture.kt @@ -1,33 +1,31 @@ -@file:OptIn(ExperimentalStdlibApi::class) - package graphics.scenery.backends.vulkan import graphics.scenery.textures.Texture import graphics.scenery.textures.Texture.BorderColor -import graphics.scenery.textures.UpdatableTexture.TextureExtents import graphics.scenery.textures.Texture.RepeatMode import graphics.scenery.textures.UpdatableTexture -import graphics.scenery.textures.UpdatableTexture.TextureUpdate import graphics.scenery.utils.Image +import graphics.scenery.utils.launchPeriodicAsync import graphics.scenery.utils.lazyLogger +import kotlinx.coroutines.* import net.imglib2.type.numeric.NumericType import net.imglib2.type.numeric.integer.* import net.imglib2.type.numeric.real.DoubleType import net.imglib2.type.numeric.real.FloatType +import org.joml.Vector4i import org.lwjgl.system.MemoryStack.stackPush import org.lwjgl.system.MemoryUtil.* import org.lwjgl.vulkan.* import org.lwjgl.vulkan.VK10.* -import org.lwjgl.vulkan.VkImageCreateInfo import java.io.FileInputStream import java.io.InputStream import java.nio.ByteBuffer import java.nio.ByteOrder -import java.nio.file.Files import java.nio.file.Paths +import kotlin.io.path.readLines import kotlin.math.max import kotlin.math.roundToLong -import kotlin.streams.toList +import kotlin.time.Duration.Companion.milliseconds /** * Vulkan Texture class. Creates a texture on the [device], with [width]x[height]x[depth], @@ -38,13 +36,44 @@ import kotlin.streams.toList * * @author Ulrik Günther */ -open class VulkanTexture(val device: VulkanDevice, - val commandPools: VulkanRenderer.CommandPools, val queue: VkQueue, val transferQueue: VkQueue, - val width: Int, val height: Int, val depth: Int = 1, - val format: Int = VK_FORMAT_R8G8B8_SRGB, var mipLevels: Int = 1, - val minFilterLinear: Boolean = true, val maxFilterLinear: Boolean = true, - val usage: HashSet = hashSetOf(Texture.UsageType.Texture)) : AutoCloseable { - //protected val logger by LazyLogger() +open class VulkanTexture( + val device: VulkanDevice, + val commandPools: VulkanRenderer.CommandPools, + val queue: VulkanDevice.QueueWithMutex, + val transferQueue: VulkanDevice.QueueWithMutex, + val width: Int, + val height: Int, + val depth: Int = 1, + val format: Int = VK_FORMAT_R8G8B8_SRGB, + var mipLevels: Int = 1, + val minFilterLinear: Boolean = true, + val maxFilterLinear: Boolean = true, + val usage: HashSet = hashSetOf(Texture.UsageType.Texture) +) : AutoCloseable { + + /** + * Alternative constructor to create a [VulkanTexture] from a [Texture]. + */ + constructor( + device: VulkanDevice, + commandPools: VulkanRenderer.CommandPools, queue: VulkanDevice.QueueWithMutex, transferQueue: VulkanDevice.QueueWithMutex, + texture: Texture, mipLevels: Int = 1) : this( + device, + commandPools, + queue, + transferQueue, + texture.dimensions.x(), + texture.dimensions.y(), + texture.dimensions.z(), + texture.toVulkanFormat(), + mipLevels, + texture.minFilter == Texture.FilteringMode.Linear, + texture.maxFilter == Texture.FilteringMode.Linear, + usage = texture.usageType + ) { + this.texture = texture + this.texture?.let { cache.put(it, this) } + } private var initialised: Boolean = false @@ -52,532 +81,267 @@ open class VulkanTexture(val device: VulkanDevice, var image: VulkanImage protected set - private var stagingImage: VulkanImage - private var gt: Texture? = null - - /** - * Wrapper class for holding on to raw Vulkan [image]s backed by [memory]. - */ - inner class VulkanImage(var image: Long = -1L, var memory: Long = -1L, val maxSize: Long = -1L) { - - /** Raw Vulkan sampler. */ - var sampler: Long = -1L - internal set - /** Raw Vulkan view. */ - var view: Long = -1L - internal set - - /** - * Copies the content of the image from [buffer]. This gets executed - * within a given [commandBuffer]. - */ - fun copyFrom(commandBuffer: VkCommandBuffer, buffer: VulkanBuffer, update: TextureUpdate? = null, bufferOffset: Long = 0) { - with(commandBuffer) { - val bufferImageCopy = VkBufferImageCopy.calloc(1) - - bufferImageCopy.imageSubresource() - .aspectMask(VK_IMAGE_ASPECT_COLOR_BIT) - .mipLevel(0) - .baseArrayLayer(0) - .layerCount(1) - bufferImageCopy.bufferOffset(bufferOffset) - - if(update != null) { - bufferImageCopy.imageExtent().set(update.extents.w, update.extents.h, update.extents.d) - bufferImageCopy.imageOffset().set(update.extents.x, update.extents.y, update.extents.z) - } else { - bufferImageCopy.imageExtent().set(width, height, depth) - bufferImageCopy.imageOffset().set(0, 0, 0) - } - - vkCmdCopyBufferToImage(this, - buffer.vulkanBuffer, - this@VulkanImage.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - bufferImageCopy) - - bufferImageCopy.free() - } - - update?.let { - it.consumed = true - } - } - - /** - * Copies the content of the image to [buffer] from a series of [updates]. This gets executed - * within a given [commandBuffer]. - */ - fun copyFrom(commandBuffer: VkCommandBuffer, buffer: VulkanBuffer, updates: List, bufferOffset: Long = 0) { - logger.debug("Got {} texture updates for {}", updates.size, this) - with(commandBuffer) { - val bufferImageCopy = VkBufferImageCopy.calloc(1) - var offset = bufferOffset - - updates.forEach { update -> - val updateSize = update.contents.remaining() - bufferImageCopy.imageSubresource() - .aspectMask(VK_IMAGE_ASPECT_COLOR_BIT) - .mipLevel(0) - .baseArrayLayer(0) - .layerCount(1) - bufferImageCopy.bufferOffset(offset) - - bufferImageCopy.imageExtent().set(update.extents.w, update.extents.h, update.extents.d) - bufferImageCopy.imageOffset().set(update.extents.x, update.extents.y, update.extents.z) - - vkCmdCopyBufferToImage(this, - buffer.vulkanBuffer, - this@VulkanImage.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - bufferImageCopy) - - offset += updateSize - update.consumed = true - } - - bufferImageCopy.free() - } - } - - /** - * Copies the content of the image from a given [VulkanImage], [image]. - * This gets executed within a given [commandBuffer]. - */ - fun copyFrom(commandBuffer: VkCommandBuffer, image: VulkanImage, extents: TextureExtents? = null) { - with(commandBuffer) { - val subresource = VkImageSubresourceLayers.calloc() - .aspectMask(VK_IMAGE_ASPECT_COLOR_BIT) - .baseArrayLayer(0) - .mipLevel(0) - .layerCount(1) - - val region = VkImageCopy.calloc(1) - .srcSubresource(subresource) - .dstSubresource(subresource) - - if(extents != null) { - region.srcOffset().set(extents.x, extents.y, extents.z) - region.dstOffset().set(extents.x, extents.y, extents.z) - region.extent().set(extents.w, extents.h, extents.d) - } else { - region.srcOffset().set(0, 0, 0) - region.dstOffset().set(0, 0, 0) - region.extent().set(width, height, depth) - } - - vkCmdCopyImage(this, - image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - this@VulkanImage.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - region) - - subresource.free() - region.free() - } - } - - override fun toString(): String { - return "VulkanImage (${this.image.toHexString()}, ${width}x${height}x${depth}, format=${format.formatToString()}, maxSize=${this.maxSize})" - } - } + /** The [Texture] this [VulkanTexture] is based on. */ + var texture: Texture? = null + internal set init { - stagingImage = if(depth == 1) { - createImage(width, height, depth, - format, VK_IMAGE_USAGE_TRANSFER_SRC_BIT, - VK_IMAGE_TILING_LINEAR, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT or VK_MEMORY_PROPERTY_HOST_CACHED_BIT, - mipLevels = 1 - ) - } else { - createImage(16, 16, 1, - format, VK_IMAGE_USAGE_TRANSFER_SRC_BIT, - VK_IMAGE_TILING_LINEAR, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT or VK_MEMORY_PROPERTY_HOST_CACHED_BIT, - mipLevels = 1) - } - var usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT or VK_IMAGE_USAGE_SAMPLED_BIT or VK_IMAGE_USAGE_TRANSFER_SRC_BIT if(device.formatFeatureSupported(format, VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT, optimalTiling = true)) { usage = usage or VK_IMAGE_USAGE_STORAGE_BIT } - image = createImage(width, height, depth, - format, usage, - VK_IMAGE_TILING_OPTIMAL, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, - mipLevels) + image = VulkanImage.create(device, width, height, depth, + format, usage, + VK_IMAGE_TILING_OPTIMAL, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, + mipLevels) if (image.sampler == -1L) { image.sampler = createSampler() } if (image.view == -1L) { - image.view = createImageView(image, format) + val swizzle = if(texture?.channels == 1 && depth > 1) { + Vector4i() + } else { + null + } + + image.view = image.createView(mipLevels, swizzle) } - gt?.let { cache.put(it, this) } + texture?.let { cache.put(it, this) } } - /** - * Alternative constructor to create a [VulkanTexture] from a [Texture]. - */ - constructor(device: VulkanDevice, - commandPools: VulkanRenderer.CommandPools, queue: VkQueue, transferQueue: VkQueue, - texture: Texture, mipLevels: Int = 1) : this(device, - commandPools, - queue, - transferQueue, - texture.dimensions.x(), - texture.dimensions.y(), - texture.dimensions.z(), - texture.toVulkanFormat(), - mipLevels, texture.minFilter == Texture.FilteringMode.Linear, texture.maxFilter == Texture.FilteringMode.Linear, usage = texture.usageType) { - gt = texture - gt?.let { cache.put(it, this) } - } + var tmpBuffer: VulkanBuffer? = null - /** - * Creates a Vulkan image of [format] with a given [width], [height], and [depth]. - * [usage] and [memoryFlags] need to be given, as well as the [tiling] parameter and number of [mipLevels]. - * A custom memory allocator may be used and given as [customAllocator]. - */ - fun createImage(width: Int, height: Int, depth: Int, format: Int, - usage: Int, tiling: Int, memoryFlags: Int, mipLevels: Int, - initialLayout: Int? = null, - customAllocator: ((VkMemoryRequirements, Long) -> Long)? = null, imageCreateInfo: VkImageCreateInfo? = null): VulkanImage { - val imageInfo = if(imageCreateInfo != null) { - imageCreateInfo - } else { - val i = VkImageCreateInfo.calloc() - .sType(VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO) - .imageType(if (depth == 1) { - VK_IMAGE_TYPE_2D - } else { - VK_IMAGE_TYPE_3D - }) - .mipLevels(mipLevels) - .arrayLayers(1) - .format(format) - .tiling(tiling) - .initialLayout(if(depth == 1) {VK_IMAGE_LAYOUT_PREINITIALIZED} else { VK_IMAGE_LAYOUT_UNDEFINED }) - .usage(usage) - .sharingMode(VK_SHARING_MODE_EXCLUSIVE) - .samples(VK_SAMPLE_COUNT_1_BIT) - .flags(VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) - - i.extent().set(width, height, depth) - i + @OptIn(ExperimentalUnsignedTypes::class) + private fun ByteBuffer.padToRGBA(texture: Texture): ByteBuffer { + logger.debug("Loading RGB texture, padding channels to 4 to fit RGBA") + val channelBytes = when (texture.type) { + is UnsignedByteType -> 1 + is ByteType -> 1 + is UnsignedShortType -> 2 + is ShortType -> 2 + is UnsignedIntType -> 4 + is IntType -> 4 + is FloatType -> 4 + is DoubleType -> 8 + else -> throw UnsupportedOperationException("Don't know how to handle textures of type ${texture.type.javaClass.simpleName}") } - if(initialLayout != null) { - imageInfo.initialLayout(initialLayout) + val storage = memAlloc(this.remaining() / 3 * 4) + val view = this.duplicate().order(ByteOrder.LITTLE_ENDIAN) + val tmp = ByteArray(channelBytes * 3) + val alpha = when(texture.type) { + is UnsignedByteType -> ubyteArrayOf(0xffu) + is ByteType -> ubyteArrayOf(0xffu) + is UnsignedShortType -> ubyteArrayOf(0xffu, 0xffu) + is ShortType -> ubyteArrayOf(0xffu, 0xffu) + is UnsignedIntType -> ubyteArrayOf(0x3fu, 0x80u, 0x00u, 0x00u) + is IntType -> ubyteArrayOf(0xffu, 0xffu, 0x00u, 0x00u) + is FloatType -> ubyteArrayOf(0x3fu, 0x80u, 0x00u, 0x00u) + is DoubleType -> ubyteArrayOf(0x3fu, 0xf0u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u) + else -> throw UnsupportedOperationException("Don't know how to handle textures of type ${texture.type.javaClass.simpleName}") } - val image = VU.getLong("create staging image", - { vkCreateImage(device.vulkanDevice, imageInfo, null, this) }, {}) + // pad buffer to 4 channels + while (view.hasRemaining()) { + view.get(tmp, 0, tmp.size) + storage.put(tmp) + storage.put(alpha.toByteArray()) + } - val reqs = VkMemoryRequirements.calloc() - vkGetImageMemoryRequirements(device.vulkanDevice, image, reqs) - val memorySize = reqs.size() + storage.flip() + return storage + } - val memory = if(customAllocator == null) { - val allocInfo = VkMemoryAllocateInfo.calloc() - .sType(VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO) - .pNext(NULL) - .allocationSize(memorySize) - .memoryTypeIndex(device.getMemoryType(reqs.memoryTypeBits(), memoryFlags).first()) + private fun isTextureDoneUpdating(tex: Texture?, fence: Long): Boolean { + val done = vkGetFenceStatus(device.vulkanDevice, fence) == VK_SUCCESS - VU.getLong("allocate image staging memory of size $memorySize", - { vkAllocateMemory(device.vulkanDevice, allocInfo, null, this) }, - { imageInfo.free(); allocInfo.free() }) - } else { - customAllocator.invoke(reqs, image) - } + if(done) { + this@VulkanTexture.device.destroyFence(fence) + // release GPU upload mutex + tex?.gpuMutex?.release() - reqs.free() + // mark as uploaded + tex?.uploaded?.incrementAndGet() - vkBindImageMemory(device.vulkanDevice, image, memory, 0) + // necessary to clear updates here, as the command buffer might still access the + // memory address of the texture update. + (tex as? UpdatableTexture)?.clearConsumedUpdates() - return VulkanImage(image, memory, memorySize) - } + // vkFreeCommandBuffers(device, commandPools.Transfer, this) + } - var tmpBuffer: VulkanBuffer? = null + return done + } /** * Copies the data for this texture from a [ByteBuffer], [data]. */ - @OptIn(ExperimentalUnsignedTypes::class) fun copyFrom(data: ByteBuffer): VulkanTexture { - if (depth == 1 && data.remaining() > stagingImage.maxSize) { - logger.warn("Allocated image size for $this (${stagingImage.maxSize}) less than copy source size ${data.remaining()}.") - return this - } - - var deallocate = false - var sourceBuffer = data - - gt?.let { gt -> - if (gt.channels == 3) { - logger.debug("Loading RGB texture, padding channels to 4 to fit RGBA") - val channelBytes = when (gt.type) { - is UnsignedByteType -> 1 - is ByteType -> 1 - is UnsignedShortType -> 2 - is ShortType -> 2 - is UnsignedIntType -> 4 - is IntType -> 4 - is FloatType -> 4 - is DoubleType -> 8 - else -> throw UnsupportedOperationException("Don't know how to handle textures of type ${gt.type.javaClass.simpleName}") - } - - val storage = memAlloc(data.remaining() / 3 * 4) - val view = data.duplicate().order(ByteOrder.LITTLE_ENDIAN) - val tmp = ByteArray(channelBytes * 3) - val alpha = when(gt.type) { - is UnsignedByteType -> ubyteArrayOf(0xffu) - is ByteType -> ubyteArrayOf(0xffu) - is UnsignedShortType -> ubyteArrayOf(0xffu, 0xffu) - is ShortType -> ubyteArrayOf(0xffu, 0xffu) - is UnsignedIntType -> ubyteArrayOf(0x3fu, 0x80u, 0x00u, 0x00u) - is IntType -> ubyteArrayOf(0xffu, 0xffu, 0x00u, 0x00u) - is FloatType -> ubyteArrayOf(0x3fu, 0x80u, 0x00u, 0x00u) - is DoubleType -> ubyteArrayOf(0x3fu, 0xf0u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u, 0x00u) - else -> throw UnsupportedOperationException("Don't know how to handle textures of type ${gt.type.javaClass.simpleName}") - } - - // pad buffer to 4 channels - while (view.hasRemaining()) { - view.get(tmp, 0, tmp.size) - storage.put(tmp) - storage.put(alpha.toByteArray()) - } - - storage.flip() - deallocate = true - sourceBuffer = storage - } else { - deallocate = false - sourceBuffer = data - } + // have an immutable reference to the texture information available + val tex = texture + + // If we have an RGB texture at hand here, we pad to RGBA due to much + // better support. In the padded case, we own the resulting buffer, which + // gets marked for deallocation after it has been uploaded to the GPU. + val (deallocate, sourceBuffer) = if(tex != null && tex.channels == 3) { + true to data.padToRGBA(tex) + } else { + false to data } - logger.debug("Updating {} with {} miplevels", this, mipLevels) - if (mipLevels == 1) { - with(VU.newCommandBuffer(device, commandPools.Standard, autostart = true)) { - if(!initialised) { - transitionLayout(stagingImage.image, - VK_IMAGE_LAYOUT_PREINITIALIZED, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, mipLevels, - srcStage = VK_PIPELINE_STAGE_HOST_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - commandBuffer = this) - } - - if (depth == 1) { - val dest = memAllocPointer(1) - vkMapMemory(device, stagingImage.memory, 0, sourceBuffer.remaining() * 1L, 0, dest) - memCopy(memAddress(sourceBuffer), dest.get(0), sourceBuffer.remaining().toLong()) - vkUnmapMemory(device, stagingImage.memory) - memFree(dest) - - transitionLayout(image.image, - VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, mipLevels, - srcStage = VK_PIPELINE_STAGE_HOST_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - commandBuffer = this) + // Textures need to be explicitly marked for async loading, so if the [Texture.usageType] + // does not contain [Texture.UsageType.AsyncLoad], loading will block. + val block = !(tex?.usageType?.contains(Texture.UsageType.AsyncLoad) ?: false) - image.copyFrom(this, stagingImage) + // acquire GPU upload mutex + tex?.gpuMutex?.acquire() - transitionLayout(image.image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, mipLevels, - srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - commandBuffer = this) + val t = CoroutineScope(TextureDispatcher).launch { + val threadLocalTransferPool = device.createCommandPool(device.queueIndices.transferQueue.first) + val threadLocalGraphicsPool = device.createCommandPool(device.queueIndices.graphicsQueue.first) + with(VU.newCommandBuffer(device, threadLocalTransferPool, autostart = true)) { + val fence = if(block) { + null } else { - val genericTexture = gt - val requiredCapacity = if(genericTexture is UpdatableTexture && genericTexture.hasConsumableUpdates()) { - genericTexture.getConsumableUpdates().map { it.contents.remaining() }.sum().toLong() - } else { - sourceBuffer.capacity().toLong() - } - - logger.debug("{} has {} consumeable updates", this@VulkanTexture, (genericTexture as? UpdatableTexture)?.getConsumableUpdates()?.size) - - if(tmpBuffer == null || (tmpBuffer?.size ?: 0) < requiredCapacity) { - logger.debug( - "({}) Reallocating tmp buffer, old size={} new size = {} MiB", - this@VulkanTexture, - tmpBuffer?.size, - requiredCapacity.toFloat()/1024.0f/1024.0f - ) - tmpBuffer?.close() - // reserve a bit more space if the texture is small, to avoid reallocations - val reservedSize = if(requiredCapacity < 1024*1024*8) { - (requiredCapacity * 1.33).roundToLong() - } else { - requiredCapacity - } + val f = this@VulkanTexture.device.createFence() - tmpBuffer = VulkanBuffer(this@VulkanTexture.device, - max(reservedSize, 1024*1024), - VK_BUFFER_USAGE_TRANSFER_SRC_BIT or VK_BUFFER_USAGE_TRANSFER_DST_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT or VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, - wantAligned = false) - } - - tmpBuffer?.let { buffer -> - transitionLayout(image.image, - VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, mipLevels, - srcStage = VK_PIPELINE_STAGE_HOST_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - commandBuffer = this) - - if(genericTexture is UpdatableTexture) { - if(genericTexture.hasConsumableUpdates()) { - val contents = genericTexture.getConsumableUpdates().map { it.contents } - - buffer.copyFrom(contents, keepMapped = true) - image.copyFrom(this, buffer, genericTexture.getConsumableUpdates()) - } /*else { - // TODO: Semantics, do we want UpdateableTextures to be only - // updateable via updates, or shall they read from buffer on first init? - buffer.copyFrom(sourceBuffer) - image.copyFrom(this, buffer) - }*/ - } else { - if(sourceBuffer.remaining() > 0) { - buffer.copyFrom(sourceBuffer) - image.copyFrom(this, buffer) - } - } - - transitionLayout(image.image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, mipLevels, - srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - commandBuffer = this) + // We are going to check every 50ms if the texture is done uploading. + CoroutineScope(TextureDispatcher).launchPeriodicAsync(50.milliseconds) { + isTextureDoneUpdating(tex, f) } + f } - endCommandBuffer(this@VulkanTexture.device, commandPools.Standard, transferQueue, flush = true, dealloc = true, block = true) - // necessary to clear updates here, as the command buffer might still access the - // memory address of the texture update. - (gt as? UpdatableTexture)?.clearConsumedUpdates() - } - } else { - val buffer = VulkanBuffer(device, - sourceBuffer.limit().toLong(), - VK_BUFFER_USAGE_TRANSFER_SRC_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, - wantAligned = false) - - with(VU.newCommandBuffer(device, commandPools.Standard, autostart = true)) { - buffer.copyFrom(sourceBuffer) - - transitionLayout(image.image, VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, commandBuffer = this, - srcStage = VK_PIPELINE_STAGE_HOST_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT) - image.copyFrom(this, buffer) - transitionLayout(image.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 1, commandBuffer = this, - srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT) - - endCommandBuffer(this@VulkanTexture.device, commandPools.Standard, transferQueue, flush = true, dealloc = true, block = true) - } - - val imageBlit = VkImageBlit.calloc(1) - with(VU.newCommandBuffer(device, commandPools.Standard, autostart = true)) mipmapCreation@{ - - for (mipLevel in 1 until mipLevels) { - imageBlit.srcSubresource().set(VK_IMAGE_ASPECT_COLOR_BIT, mipLevel - 1, 0, 1) - imageBlit.srcOffsets(1).set(width shr (mipLevel - 1), height shr (mipLevel - 1), 1) - - val dstWidth = width shr mipLevel - val dstHeight = height shr mipLevel - - if (dstWidth < 2 || dstHeight < 2) { - break + val requiredCapacity = + if (tex is UpdatableTexture && tex.hasConsumableUpdates()) { + tex.getConsumableUpdates().sumOf { it.contents.remaining() }.toLong() + } else { + sourceBuffer.remaining().toLong() } - imageBlit.dstSubresource().set(VK_IMAGE_ASPECT_COLOR_BIT, mipLevel, 0, 1) - imageBlit.dstOffsets(1).set(width shr (mipLevel), height shr (mipLevel), 1) - - val mipSourceRange = VkImageSubresourceRange.calloc() - .aspectMask(VK_IMAGE_ASPECT_COLOR_BIT) - .baseArrayLayer(0) - .layerCount(1) - .baseMipLevel(mipLevel - 1) - .levelCount(1) + logger.debug( + "{} has {} consumable updates", + this@VulkanTexture, + (tex as? UpdatableTexture)?.getConsumableUpdates()?.size + ) - val mipTargetRange = VkImageSubresourceRange.calloc() - .aspectMask(VK_IMAGE_ASPECT_COLOR_BIT) - .baseArrayLayer(0) - .layerCount(1) - .baseMipLevel(mipLevel) - .levelCount(1) - - if (mipLevel > 1) { - transitionLayout(image.image, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - subresourceRange = mipSourceRange, - srcStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - commandBuffer = this@mipmapCreation) + if(tmpBuffer == null || (tmpBuffer?.size ?: 0) < requiredCapacity) { + logger.debug( + "({}) Reallocating tmp buffer, old size={} new size = {} MiB", + this@VulkanTexture, + tmpBuffer?.size, + requiredCapacity.toFloat()/1024.0f/1024.0f + ) + + tmpBuffer?.close() + // reserve a bit more space if the texture is small, to avoid reallocations + val reservedSize = if(tex is UpdatableTexture && requiredCapacity < 1024*1024*8) { + (requiredCapacity * 1.33).roundToLong() + } else { + requiredCapacity } - transitionLayout(image.image, + tmpBuffer = VulkanBuffer( + this@VulkanTexture.device, + max(reservedSize, 1024 * 1024), + VK_BUFFER_USAGE_TRANSFER_SRC_BIT or VK_BUFFER_USAGE_TRANSFER_DST_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT or VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, + wantAligned = false + ) + } + + tmpBuffer?.let { buffer -> + transitionLayout( + image.image, VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - subresourceRange = mipTargetRange, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, mipLevels, srcStage = VK_PIPELINE_STAGE_HOST_BIT, dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - commandBuffer = this@mipmapCreation) - - vkCmdBlitImage(this@mipmapCreation, - image.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - image.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - imageBlit, VK_FILTER_LINEAR) - - transitionLayout(image.image, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, subresourceRange = mipSourceRange, - srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - commandBuffer = this@mipmapCreation) + commandBuffer = this + ) + + // An updatable texture can be either filled with it's updates, + // or, if it's loaded for the first time, with it's default + // byte buffer contents + if(tex is UpdatableTexture && tex.hasConsumableUpdates()) { + val contents = tex.getConsumableUpdates().map { it.contents } + + tex.mutex.acquire() + buffer.copyFrom(contents, keepMapped = true) + image.copyFrom(this, buffer, tex.getConsumableUpdates()) + tex.mutex.release() + } else { + tex?.mutex?.acquire() + buffer.copyFrom(sourceBuffer) + image.copyFrom(this, buffer) + tex?.mutex?.release() + } - transitionLayout(image.image, + transitionLayout( + image.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, subresourceRange = mipTargetRange, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, mipLevels, srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - commandBuffer = this@mipmapCreation) + dstStage = VK_PIPELINE_STAGE_HOST_BIT, + commandBuffer = this + ) - mipSourceRange.free() - mipTargetRange.free() } - this@mipmapCreation.endCommandBuffer(this@VulkanTexture.device, commandPools.Standard, queue, flush = true, dealloc = true) + endCommandBuffer( + this@VulkanTexture.device, + threadLocalTransferPool, + transferQueue, + flush = true, + // FIXME: make deallocation work again when running async + dealloc = false, + block = block, + fence = fence + ) } - imageBlit.free() - buffer.close() + if(mipLevels > 1) { + with(VU.newCommandBuffer(device, threadLocalGraphicsPool, autostart = true)) { + logger.debug("Updating {} with {} miplevels", this, mipLevels) + // maybeCreateMipmaps will immediately return if there's only one level requested + image.maybeCreateMipmaps(this, mipLevels) + + endCommandBuffer( + this@VulkanTexture.device, + threadLocalGraphicsPool, + queue, + flush = true, + // FIXME: make deallocation work again when running async + dealloc = false, + block = block + ) + } + } } + if(block) { + runBlocking { t.join() } + tex?.gpuMutex?.release() + + // necessary to clear updates here, as the command buffer might still access the + // memory address of the texture update. + (tex as? UpdatableTexture)?.clearConsumedUpdates() + } + // necessary to clear updates here, as the command buffer might still access the + // memory address of the texture update. + (tex as? UpdatableTexture)?.clearConsumedUpdates() + // deallocate in case we moved pixels around if (deallocate) { memFree(sourceBuffer) } -// image.view = createImageView(image, format) - initialised = true return this } @@ -585,91 +349,111 @@ open class VulkanTexture(val device: VulkanDevice, /** * Copies the first layer, first mipmap of the texture to [buffer]. */ - fun copyTo(buffer: ByteBuffer) { - if(tmpBuffer == null || (tmpBuffer?.size!! < image.maxSize)) { - tmpBuffer?.close() - tmpBuffer = VulkanBuffer(this@VulkanTexture.device, - image.maxSize, - VK_BUFFER_USAGE_TRANSFER_SRC_BIT or VK_BUFFER_USAGE_TRANSFER_DST_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT or VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, - wantAligned = false) - } + fun copyTo(buffer: ByteBuffer, inPlace: Boolean = false): ByteBuffer? { + stackPush().use { stack -> + val memoryProperties = (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT + or VK_MEMORY_PROPERTY_HOST_COHERENT_BIT + or VK_MEMORY_PROPERTY_HOST_CACHED_BIT) + + if (tmpBuffer == null + || (tmpBuffer?.size!! < image.maxSize) + || (tmpBuffer?.requestedMemoryProperties != memoryProperties)) { + tmpBuffer?.close() + logger.debug("Reallocating temporary buffer") + tmpBuffer = VulkanBuffer(this@VulkanTexture.device, + image.maxSize, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT or VK_BUFFER_USAGE_TRANSFER_DST_BIT, + memoryProperties, + wantAligned = true) + } - tmpBuffer?.let { b -> - with(VU.newCommandBuffer(device, commandPools.Standard, autostart = true)) { - transitionLayout(image.image, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 1, - srcStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - dstStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - commandBuffer = this) - - val type = VK_IMAGE_ASPECT_COLOR_BIT - - val subresource = VkImageSubresourceLayers.calloc() - .aspectMask(type) - .mipLevel(0) - .baseArrayLayer(0) - .layerCount(1) - - val regions = VkBufferImageCopy.calloc(1) - .bufferRowLength(0) - .bufferImageHeight(0) - .imageOffset(VkOffset3D.calloc().set(0, 0, 0)) - .imageExtent(VkExtent3D.calloc().set(width, height, depth)) - .imageSubresource(subresource) - - vkCmdCopyImageToBuffer( - this, - image.image, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - b.vulkanBuffer, - regions - ) + tmpBuffer?.let { b -> + with(VU.newCommandBuffer(device, commandPools.Transfer, autostart = true)) { + logger.info("${System.nanoTime()}: Copying $width $height $depth") + transitionLayout(image.image, + from = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + to = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + srcStage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + dstStage = VK_PIPELINE_STAGE_HOST_BIT, + dstAccessMask = VK_ACCESS_HOST_READ_BIT, + commandBuffer = this) - transitionLayout(image.image, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 1, - srcStage = VK_PIPELINE_STAGE_TRANSFER_BIT, - dstStage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - commandBuffer = this) + val type = VK_IMAGE_ASPECT_COLOR_BIT - endCommandBuffer(this@VulkanTexture.device, commandPools.Standard, transferQueue, flush = true, dealloc = true, block = true) - } + val subresource = VkImageSubresourceLayers.calloc(stack) + .aspectMask(type) + .mipLevel(0) + .baseArrayLayer(0) + .layerCount(1) - b.copyTo(buffer) - } - } + val regions = VkBufferImageCopy.calloc(1, stack) + .bufferRowLength(0) + .bufferImageHeight(0) + .imageOffset(VkOffset3D.calloc(stack).set(0, 0, 0)) + .imageExtent(VkExtent3D.calloc(stack).set(width, height, depth)) + .imageSubresource(subresource) - /** - * Creates a Vulkan image view with [format] for an [image]. - */ - fun createImageView(image: VulkanImage, format: Int): Long { - if(image.view != -1L) { - vkDestroyImageView(device.vulkanDevice, image.view, null) - } + vkCmdCopyImageToBuffer( + this, + image.image, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + b.vulkanBuffer, + regions + ) - val subresourceRange = VkImageSubresourceRange.calloc().set(VK_IMAGE_ASPECT_COLOR_BIT, 0, mipLevels, 0, 1) + transitionLayout(image.image, + from = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + to = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + srcStage = VK_PIPELINE_STAGE_HOST_BIT, + srcAccessMask = VK_ACCESS_HOST_READ_BIT, + dstStage = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT, + dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + commandBuffer = this) - val vi = VkImageViewCreateInfo.calloc() - .sType(VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO) - .pNext(NULL) - .image(image.image) - .viewType(if (depth > 1) { - VK_IMAGE_VIEW_TYPE_3D - } else { - VK_IMAGE_VIEW_TYPE_2D - }) - .format(format) - .subresourceRange(subresourceRange) + val barrier = VkBufferMemoryBarrier.calloc(1, stack) + barrier + .sType(VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER) + .buffer(b.vulkanBuffer) + .srcAccessMask(VK_ACCESS_TRANSFER_WRITE_BIT) + .dstAccessMask(VK_ACCESS_HOST_READ_BIT) + .size(VK_WHOLE_SIZE) + + vkCmdPipelineBarrier( + this, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_HOST_BIT, + 0, + null, + barrier, + null + ) + + endCommandBuffer( + this@VulkanTexture.device, + commandPools.Transfer, + transferQueue, + flush = true, + dealloc = true, + block = true + ) + } + + val result = if(!inPlace) { + b.copyTo(buffer) + buffer + } else { + b.unmap() + val p = b.map() + logger.info("${System.nanoTime()}: Buffer has ${buffer.remaining()}") + memByteBuffer(p.get(0), buffer.remaining()) + } - if(gt?.channels == 1 && depth > 1) { - vi.components().set(VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_R) + return result + } } - return VU.getLong("Creating image view", - { vkCreateImageView(device.vulkanDevice, vi, null, this) }, - { vi.free(); subresourceRange.free(); }) + return null } private fun RepeatMode.toVulkan(): Int { @@ -699,7 +483,7 @@ open class VulkanTexture(val device: VulkanDevice, * Creates a default sampler for this texture. */ fun createSampler(texture: Texture? = null): Long { - val t = texture ?: gt + val t = texture ?: this.texture val (repeatS, repeatT, repeatU) = if(t != null) { Triple( @@ -750,7 +534,7 @@ open class VulkanTexture(val device: VulkanDevice, } override fun toString(): String { - return "VulkanTexture on $device (${this.image.image.toHexString()}, ${width}x${height}x$depth, format=${this.format.formatToString()}, mipLevels=${mipLevels}, gt=${this.gt != null} minFilter=${this.minFilterLinear} maxFilter=${this.maxFilterLinear})" + return "VulkanTexture on $device (${this.image.image.toHexString()}, ${width}x${height}x$depth, format=${this.format.formatToString()}, mipLevels=${mipLevels}, texture=${this.texture != null} minFilter=${this.minFilterLinear} maxFilter=${this.maxFilterLinear})" } /** @@ -758,7 +542,7 @@ open class VulkanTexture(val device: VulkanDevice, * related to it. */ override fun close() { - gt?.let { cache.remove(it) } + texture?.let { cache.remove(it) } if (image.view != -1L) { vkDestroyImageView(device.vulkanDevice, image.view, null) @@ -780,16 +564,6 @@ open class VulkanTexture(val device: VulkanDevice, image.memory = -1L } - if (stagingImage.image != -1L) { - vkDestroyImage(device.vulkanDevice, stagingImage.image, null) - stagingImage.image = -1L - } - - if (stagingImage.memory != -1L) { - vkFreeMemory(device.vulkanDevice, stagingImage.memory, null) - stagingImage.memory = -1L - } - tmpBuffer?.close() } @@ -801,6 +575,7 @@ open class VulkanTexture(val device: VulkanDevice, @JvmStatic private val logger by lazyLogger() private val cache = HashMap() + private val TextureDispatcher = newFixedThreadPoolContext(4, "VulkanTextureWorker") fun getReference(texture: Texture): VulkanTexture? { return cache.get(texture) @@ -810,7 +585,7 @@ open class VulkanTexture(val device: VulkanDevice, * Loads a texture from a file given by [filename], and allocates the [VulkanTexture] on [device]. */ fun loadFromFile(device: VulkanDevice, - commandPools: VulkanRenderer.CommandPools , queue: VkQueue, transferQueue: VkQueue, + commandPools: VulkanRenderer.CommandPools , queue: VulkanDevice.QueueWithMutex, transferQueue: VulkanDevice.QueueWithMutex, filename: String, linearMin: Boolean, linearMax: Boolean, generateMipmaps: Boolean = true): VulkanTexture { @@ -823,7 +598,7 @@ open class VulkanTexture(val device: VulkanDevice, return if(type == "raw") { val path = Paths.get(filename) val infoFile = path.resolveSibling(path.fileName.toString().substringBeforeLast(".") + ".info") - val dimensions = Files.lines(infoFile).toList().first().split(",").map { it.toLong() }.toLongArray() + val dimensions = infoFile.readLines().first().split(",").map { it.toLong() }.toLongArray() loadFromFileRaw(device, commandPools, queue, transferQueue, @@ -839,7 +614,7 @@ open class VulkanTexture(val device: VulkanDevice, * Loads a texture from a file given by a [stream], and allocates the [VulkanTexture] on [device]. */ fun loadFromFile(device: VulkanDevice, - commandPools: VulkanRenderer.CommandPools, queue: VkQueue, transferQueue: VkQueue, + commandPools: VulkanRenderer.CommandPools, queue: VulkanDevice.QueueWithMutex, transferQueue: VulkanDevice.QueueWithMutex, stream: InputStream, type: String, linearMin: Boolean, linearMax: Boolean, generateMipmaps: Boolean = true): VulkanTexture { @@ -883,7 +658,7 @@ open class VulkanTexture(val device: VulkanDevice, */ @Suppress("UNUSED_PARAMETER") fun loadFromFileRaw(device: VulkanDevice, - commandPools: VulkanRenderer.CommandPools, queue: VkQueue, transferQueue: VkQueue, + commandPools: VulkanRenderer.CommandPools, queue: VulkanDevice.QueueWithMutex, transferQueue: VulkanDevice.QueueWithMutex, stream: InputStream, type: String, dimensions: LongArray): VulkanTexture { val imageData: ByteBuffer = ByteBuffer.allocateDirect((2 * dimensions[0] * dimensions[1] * dimensions[2]).toInt()) val buffer = ByteArray(1024*1024) @@ -913,17 +688,24 @@ open class VulkanTexture(val device: VulkanDevice, */ fun transitionLayout(image: Long, from: Int, to: Int, mipLevels: Int = 1, subresourceRange: VkImageSubresourceRange? = null, - srcStage: Int = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, dstStage: Int = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - srcAccessMask: Int, dstAccessMask: Int, - commandBuffer: VkCommandBuffer, dependencyFlags: Int = 0, memoryBarrier: Boolean = false) { + srcStage: Int = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + dstStage: Int = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + srcAccessMask: Int, + dstAccessMask: Int, + commandBuffer: VkCommandBuffer, + dependencyFlags: Int = 0, + memoryBarrier: Boolean = false, + srcQueueFamilyIndex: Int = VK_QUEUE_FAMILY_IGNORED, + dstQueueFamilyIndex: Int = VK_QUEUE_FAMILY_IGNORED + ) { stackPush().use { stack -> val barrier = VkImageMemoryBarrier.calloc(1, stack) .sType(VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER) .pNext(NULL) .oldLayout(from) .newLayout(to) - .srcQueueFamilyIndex(VK_QUEUE_FAMILY_IGNORED) - .dstQueueFamilyIndex(VK_QUEUE_FAMILY_IGNORED) + .srcQueueFamilyIndex(srcQueueFamilyIndex) + .dstQueueFamilyIndex(dstQueueFamilyIndex) .srcAccessMask(srcAccessMask) .dstAccessMask(dstAccessMask) .image(image) diff --git a/src/main/kotlin/graphics/scenery/compute/EdgeBundler.kt b/src/main/kotlin/graphics/scenery/compute/EdgeBundler.kt index f5dd2a3acb..9e82083592 100644 --- a/src/main/kotlin/graphics/scenery/compute/EdgeBundler.kt +++ b/src/main/kotlin/graphics/scenery/compute/EdgeBundler.kt @@ -214,7 +214,7 @@ class EdgeBundler(override var hub: Hub?): Hubable { line.name = trackId.toString() line.material { blending.opacity = paramAlpha - depthTest = Material.DepthTest.Always + depthOp = Material.DepthTest.Always } line.spatial { position = Vector3f(0.0f, 0.0f, 0.0f) diff --git a/src/main/kotlin/graphics/scenery/controls/GLFWMouseAndKeyHandler.kt b/src/main/kotlin/graphics/scenery/controls/GLFWMouseAndKeyHandler.kt index 83bb3dafdd..7970e677de 100644 --- a/src/main/kotlin/graphics/scenery/controls/GLFWMouseAndKeyHandler.kt +++ b/src/main/kotlin/graphics/scenery/controls/GLFWMouseAndKeyHandler.kt @@ -93,12 +93,21 @@ open class GLFWMouseAndKeyHandler(var hub: Hub?) : MouseAndKeyHandlerBase(), Aut else -> KeyEvent.KEY_PRESSED } + // Fix cursor key mapping + val mappedKey = when(key) { + GLFW_KEY_UP -> KeyEvent.VK_UP + GLFW_KEY_DOWN -> KeyEvent.VK_DOWN + GLFW_KEY_LEFT -> KeyEvent.VK_LEFT + GLFW_KEY_RIGHT -> KeyEvent.VK_RIGHT + else -> key + } + val event = KeyEvent( fakeComponent, type, System.nanoTime(), mods.glfwToSwingMods(), - key, + mappedKey, KeyEvent.CHAR_UNDEFINED ) diff --git a/src/main/kotlin/graphics/scenery/controls/Hololens.kt b/src/main/kotlin/graphics/scenery/controls/Hololens.kt index b069d13ae4..4c3cc86f2a 100644 --- a/src/main/kotlin/graphics/scenery/controls/Hololens.kt +++ b/src/main/kotlin/graphics/scenery/controls/Hololens.kt @@ -56,7 +56,7 @@ class Hololens: TrackerInput, Display, Hubable { data class CommandBufferWithStatus(val commandBuffer: VulkanCommandBuffer, var current: Boolean = false) private var commandBuffers: MutableList = mutableListOf() private var hololensCommandPool = -1L - private var d3dImages: List?> = emptyList() + private var d3dImages: List?> = emptyList() private var currentImageIndex: Int = 0 private val acqKeys = memAllocLong(1).put(0, 0) @@ -226,7 +226,7 @@ class Hololens: TrackerInput, Display, Hubable { * @param[queue] The Vulkan command queue to use. * @param[commandPool] The Vulkan command pool to use. */ - private fun getSharedHandleVulkanTexture(sharedHandleAddress: Long, width: Int, height: Int, format: Int, device: VulkanDevice, queue: VkQueue, commandPool: Long): Pair? { + private fun getSharedHandleVulkanTexture(sharedHandleAddress: Long, width: Int, height: Int, format: Int, device: VulkanDevice, queue: VulkanDevice.QueueWithMutex, commandPool: Long): Pair? { logger.info("Registered D3D shared texture handle as ${sharedHandleAddress.toHexString()}/${sharedHandleAddress.toString(16)}") // VK_EXTERNAL_MEMORY_HANDLE_TYPE_D3D11_IMAGE_BIT_NV does not seem to work here @@ -285,10 +285,6 @@ class Hololens: TrackerInput, Display, Hubable { dedicatedAllocationCreateInfo.dedicatedAllocation(true) } - val t = VulkanTexture(device, VulkanRenderer.CommandPools(commandPool, commandPool, commandPool, commandPool), queue, queue, - width, height, 1, - format, 1, true, true) - val imageCreateInfo = VkImageCreateInfo.calloc() .sType(VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO) .pNext(extMemoryImageInfo.address()) @@ -306,7 +302,7 @@ class Hololens: TrackerInput, Display, Hubable { imageCreateInfo.extent().set(hololensDisplaySize.x().toInt(), hololensDisplaySize.y().toInt(), 1) var memoryHandle: Long = -1L - val img = t.createImage(hololensDisplaySize.x().toInt(), hololensDisplaySize.y().toInt(), 1, + val img = VulkanImage.create(device, hololensDisplaySize.x().toInt(), hololensDisplaySize.y().toInt(), 1, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_USAGE_SAMPLED_BIT, VK_IMAGE_TILING_OPTIMAL, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 1, imageCreateInfo = imageCreateInfo, @@ -376,9 +372,9 @@ class Hololens: TrackerInput, Display, Hubable { * @param[queue] Vulkan queue * @param[image] The Vulkan texture image to be presented to the compositor */ - override fun submitToCompositorVulkan(width: Int, height: Int, format: Int, instance: VkInstance, device: VulkanDevice, queue: VkQueue, image: Long) { + override fun submitToCompositorVulkan(width: Int, height: Int, format: Int, instance: VkInstance, device: VulkanDevice, queue: VulkanDevice.QueueWithMutex, image: Long) { if(hololensCommandPool == -1L) { - hololensCommandPool = device.createCommandPool(device.queues.graphicsQueue.first) + hololensCommandPool = device.createCommandPool(device.queueIndices.graphicsQueue.first) } if(leftProjection == null) { diff --git a/src/main/kotlin/graphics/scenery/controls/MouseAndKeyHandlerBase.kt b/src/main/kotlin/graphics/scenery/controls/MouseAndKeyHandlerBase.kt index 21a5a23477..845b7c6a5e 100644 --- a/src/main/kotlin/graphics/scenery/controls/MouseAndKeyHandlerBase.kt +++ b/src/main/kotlin/graphics/scenery/controls/MouseAndKeyHandlerBase.kt @@ -7,7 +7,7 @@ import graphics.scenery.backends.SceneryWindow import graphics.scenery.controls.behaviours.GamepadBehaviour import graphics.scenery.controls.behaviours.GamepadClickBehaviour import graphics.scenery.utils.ExtractsNatives -import graphics.scenery.utils.ExtractsNatives.Platform.* +import graphics.scenery.utils.ExtractsNatives.Companion.extractLibrariesFromClasspath import graphics.scenery.utils.lazyLogger import net.java.games.input.* import org.lwjgl.system.Platform @@ -133,22 +133,16 @@ open class MouseAndKeyHandlerBase : ControllerListener, ExtractsNatives { init { java.util.logging.Logger.getLogger(ControllerEnvironment::class.java.name).parent.level = Level.SEVERE - /** Returns the name of the DLL/so/dylib required by JInput on the given platform. */ - fun ExtractsNatives.Platform.getPlatformJinputLibraryName(): String { - return when(this) { - WINDOWS -> "jinput-raw_64.dll" - LINUX -> "libjinput-linux64.so" - MACOS -> "libjinput-osx.jnilib" - UNKNOWN -> "none" - } - } - // JInput is not available on ARM32/64 if(Platform.getArchitecture() == Platform.Architecture.X64) { try { - val platformJars = getNativeJars("jinput-platform", hint = ExtractsNatives.getPlatform().getPlatformJinputLibraryName()) - logger.debug("Native JARs for JInput: ${platformJars.joinToString(", ")}") - val path = extractLibrariesFromJar(platformJars, load = false) + val nativeLibraries = when(Platform.get()) { + Platform.LINUX -> listOf("libjinput-linux64.so") + Platform.MACOSX -> listOf("libjinput-osx.jnilib") + Platform.WINDOWS -> listOf("jinput-raw_64.dll", "jinput-dx8_64.dll", "jinput-wintab.dll") + } + + val path = extractLibrariesFromClasspath(nativeLibraries, load = false) System.setProperty("net.java.games.input.librarypath", path) ControllerEnvironment.getDefaultEnvironment().controllers.forEach { @@ -160,10 +154,10 @@ open class MouseAndKeyHandlerBase : ControllerListener, ExtractsNatives { } catch(ule: UnsatisfiedLinkError) { logger.warn("Could not initialize JInput due to an UnsatisfiedLinkError: ${ule.message}") logger.warn("This could be to either your platform not being supported by JInput, or the JInput natives missing from the classpath.") - logger.debug("Traceback: ${ule.stackTrace}") + logger.debug("Traceback: {}", ule.stackTrace) } catch (e: Exception) { logger.warn("Could not initialize JInput: ${e.message}") - logger.debug("Traceback: ${e.stackTrace}") + logger.debug("Traceback: {}", e.stackTrace) } controllerThread = thread { @@ -186,7 +180,7 @@ open class MouseAndKeyHandlerBase : ControllerListener, ExtractsNatives { val b = gamepad.behaviour if (b is GamepadBehaviour) { if (abs(it.value) > 0.02f && b.axis.contains(it.key)) { - logger.trace("Triggering ${it.key} because axis is down (${it.value})") + logger.trace("Triggering {} because axis is down ({})", it.key, it.value) b.axisEvent(it.key, it.value) } } @@ -357,11 +351,11 @@ open class MouseAndKeyHandlerBase : ControllerListener, ExtractsNatives { * @param[event] The incoming controller event */ fun controllerEvent(event: Event) { - logger.trace("Event: $event/identifier=${event.component.identifier}") + logger.trace("Event: {}/identifier={}", event, event.component.identifier) for (gamepad in gamepads) { if (event.component.isAnalog) { if (abs(event.component.pollData) < CONTROLLER_DOWN_THRESHOLD) { - logger.trace("${event.component.identifier} over threshold, removing") + logger.trace("{} over threshold, removing", event.component.identifier) controllerAxisDown[event.component.identifier] = 0.0f } else { controllerAxisDown[event.component.identifier] = event.component.pollData diff --git a/src/main/kotlin/graphics/scenery/controls/OpenVRHMD.kt b/src/main/kotlin/graphics/scenery/controls/OpenVRHMD.kt index a4c7b2e56d..6de3f28280 100644 --- a/src/main/kotlin/graphics/scenery/controls/OpenVRHMD.kt +++ b/src/main/kotlin/graphics/scenery/controls/OpenVRHMD.kt @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.KotlinModule import graphics.scenery.* import graphics.scenery.backends.Display @@ -699,7 +700,7 @@ open class OpenVRHMD(val seated: Boolean = false, val useCompositor: Boolean = t override fun submitToCompositorVulkan(width: Int, height: Int, format: Int, instance: VkInstance, device: VulkanDevice, - queue: VkQueue, image: Long) { + queue: VulkanDevice.QueueWithMutex, image: Long) { // update() if (disableSubmission || !readyForSubmission) { return @@ -711,8 +712,8 @@ open class OpenVRHMD(val seated: Boolean = false, val useCompositor: Boolean = t .m_pInstance(instance.address()) .m_pPhysicalDevice(device.physicalDevice.address()) .m_pDevice(device.vulkanDevice.address()) - .m_pQueue(queue.address()) - .m_nQueueFamilyIndex(device.queues.graphicsQueue.first) + .m_pQueue(queue.queue.address()) + .m_nQueueFamilyIndex(device.queueIndices.graphicsQueue.first) .m_nWidth(width) .m_nHeight(height) .m_nFormat(format) @@ -726,7 +727,7 @@ open class OpenVRHMD(val seated: Boolean = false, val useCompositor: Boolean = t readyForSubmission = false if(commandPool == -1L) { - commandPool = device.createCommandPool(device.queues.graphicsQueue.first) + commandPool = device.createCommandPool(device.queueIndices.graphicsQueue.first) } val subresourceRange = VkImageSubresourceRange.calloc(stack) @@ -917,61 +918,6 @@ open class OpenVRHMD(val seated: Boolean = false, val useCompositor: Boolean = t return Matrix4f(this.m()) } - private fun loadMeshFromModelPath(type: TrackedDeviceType, path: String, mesh: Mesh): Mesh { - val compositeFile = File(path.substringBeforeLast(".") + ".json") - - when { - compositeFile.exists() && compositeFile.length() > 1024 -> { - logger.info("Loading model from composite JSON, ${compositeFile.absolutePath}") - val mapper = ObjectMapper(YAMLFactory()) - mapper.registerModule(KotlinModule()) - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false) - - try { - // SteamVR's JSON contains tabs, while it shouldn't. If not replacing this, jackson will freak out. - val json = compositeFile.readText().replace("\t", " ") - val model = mapper.readValue(json, CompositeModel::class.java) - model.components.forEach { (_, component) -> - if(component.filename != null) { - val m = Mesh() - m.readFromOBJ(compositeFile.resolveSibling(component.filename).absolutePath, true) - mesh.addChild(m) - - if(component.visibility?.getOrDefault("default", true) == false) { - m.visible = false - } - } - } - } catch(e: Exception) { - logger.error("Exception: $e") - logger.info("Loading composite JSON failed, trying to fall back to regular model.") - mesh.readFrom(path) - e.printStackTrace() - } - } - - mesh.name.lowercase().endsWith("stl") || - mesh.name.lowercase().endsWith("obj") -> { - mesh.readFrom(path) - - if (type == TrackedDeviceType.Controller) { - mesh.ifMaterial { - diffuse = Vector3f(0.1f, 0.1f, 0.1f) - } - mesh.children.forEach { c -> - c.ifMaterial { - diffuse = Vector3f(0.1f, 0.1f, 0.1f) - } - } - } - } - else -> logger.warn("Unknown model format: $path for $type") - } - - - return mesh - } /** * Loads a model representing the [TrackedDevice]. @@ -1313,5 +1259,74 @@ open class OpenVRHMD(val seated: Boolean = false, val useCompositor: Boolean = t private fun HmdVector3.toVector3f(): Vector3f { return Vector3f(this.v()) } + + /** + * Loads a model for a given device [type] from the JSON file given as [path]. The + * model data loaded from the JSON file will become a child of [mesh], and [mesh] will + * again be returned by this function. + */ + fun loadMeshFromModelPath(type: TrackedDeviceType, path: String, mesh: Mesh): Mesh { + val compositeFile = File(path.substringBeforeLast(".") + ".json") + + when { + compositeFile.exists() && compositeFile.length() > 1024 -> { + logger.info("Loading model from composite JSON, ${compositeFile.absolutePath}") + val mapper = ObjectMapper(YAMLFactory()) + mapper.registerModule( + KotlinModule.Builder() + .configure(KotlinFeature.NullToEmptyCollection, true) + .configure(KotlinFeature.NullToEmptyMap, true) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, true) + .configure(KotlinFeature.StrictNullChecks, true) + .build() + ) + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + mapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false) + + try { + // SteamVR's JSON contains tabs, while it shouldn't. If not replacing this, jackson will freak out. + val json = compositeFile.readText().replace("\t", " ") + val model = mapper.readValue(json, CompositeModel::class.java) + model.components.forEach { (_, component) -> + if(component.filename != null) { + val m = Mesh() + m.readFromOBJ(compositeFile.resolveSibling(component.filename).absolutePath, true) + mesh.addChild(m) + + if(component.visibility?.getOrDefault("default", true) == false) { + m.visible = false + } + } + } + } catch(e: Exception) { + logger.error("Exception: $e") + logger.info("Loading composite JSON failed, trying to fall back to regular model.") + mesh.readFrom(path) + e.printStackTrace() + } + } + + mesh.name.lowercase().endsWith("stl") || + mesh.name.lowercase().endsWith("obj") -> { + mesh.readFrom(path) + + if (type == TrackedDeviceType.Controller) { + mesh.ifMaterial { + diffuse = Vector3f(0.1f, 0.1f, 0.1f) + } + mesh.children.forEach { c -> + c.ifMaterial { + diffuse = Vector3f(0.1f, 0.1f, 0.1f) + } + } + } + } + else -> logger.warn("Unknown model format: $path for $type") + } + + + return mesh + } } } diff --git a/src/main/kotlin/graphics/scenery/controls/ScreenConfig.kt b/src/main/kotlin/graphics/scenery/controls/ScreenConfig.kt index feeef21fc6..8f5b9d2d9b 100644 --- a/src/main/kotlin/graphics/scenery/controls/ScreenConfig.kt +++ b/src/main/kotlin/graphics/scenery/controls/ScreenConfig.kt @@ -5,6 +5,7 @@ import org.joml.Vector3f import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.KotlinFeature import com.fasterxml.jackson.module.kotlin.KotlinModule import graphics.scenery.utils.JsonDeserialisers import graphics.scenery.utils.lazyLogger @@ -142,7 +143,15 @@ class ScreenConfig { */ @JvmStatic fun loadFromFile(path: String): ScreenConfig.Config { val mapper = ObjectMapper(YAMLFactory()) - mapper.registerModule(KotlinModule()) + mapper.registerModule( + KotlinModule.Builder() + .configure(KotlinFeature.NullToEmptyCollection, true) + .configure(KotlinFeature.NullToEmptyMap, true) + .configure(KotlinFeature.NullIsSameAsDefault, false) + .configure(KotlinFeature.SingletonSupport, true) + .configure(KotlinFeature.StrictNullChecks, true) + .build() + ) var stream = ScreenConfig::class.java.getResourceAsStream(path) diff --git a/src/main/kotlin/graphics/scenery/controls/TrackedStereoGlasses.kt b/src/main/kotlin/graphics/scenery/controls/TrackedStereoGlasses.kt index 794c7b9490..51af38e17e 100644 --- a/src/main/kotlin/graphics/scenery/controls/TrackedStereoGlasses.kt +++ b/src/main/kotlin/graphics/scenery/controls/TrackedStereoGlasses.kt @@ -158,7 +158,7 @@ class TrackedStereoGlasses(var address: String = "device@localhost:5500", var sc * @param[queueFamilyIndex] Queue family index * @param[image] The Vulkan texture image to be presented to the compositor */ - override fun submitToCompositorVulkan(width: Int, height: Int, format: Int, instance: VkInstance, device: VulkanDevice, queue: VkQueue, image: Long) { + override fun submitToCompositorVulkan(width: Int, height: Int, format: Int, instance: VkInstance, device: VulkanDevice, queue: VulkanDevice.QueueWithMutex, image: Long) { logger.error("This Display implementation does not have a compositor. Incorrect configuration?") } diff --git a/src/main/kotlin/graphics/scenery/controls/behaviours/ArcballCameraControl.kt b/src/main/kotlin/graphics/scenery/controls/behaviours/ArcballCameraControl.kt index 5d262dfff3..c3db98d962 100644 --- a/src/main/kotlin/graphics/scenery/controls/behaviours/ArcballCameraControl.kt +++ b/src/main/kotlin/graphics/scenery/controls/behaviours/ArcballCameraControl.kt @@ -114,20 +114,28 @@ open class ArcballCameraControl(private val name: String, camera: () -> Camera?, lastX = x lastY = y - val frameYaw = (xoffset) / 180.0f * Math.PI.toFloat() - val framePitch = yoffset / 180.0f * Math.PI.toFloat() - - // first calculate the total rotation quaternion to be applied to the camera - val yawQ = Quaternionf().rotateXYZ(0.0f, frameYaw, 0.0f).normalize() - val pitchQ = Quaternionf().rotateXYZ(framePitch, 0.0f, 0.0f).normalize() - - node.ifSpatial { - distance = (target.invoke() - position).length() - node.target = target.invoke() - rotation = pitchQ.mul(rotation).mul(yawQ).normalize() - position = target.invoke() + node.forward * distance * (-1.0f) + node.spatial().rotateAround(xoffset, yoffset, target.invoke()) + + node.lock.unlock() + } + } + + + /** + * This function rotates the camera controlled by this behaviour by a fixed [yaw] + * and [pitch] about the [target] + * + * @param[yaw] yaw in degrees + * @param[pitch] pitch in degrees + */ + fun rotateDegrees(yaw: Float, pitch: Float) { + cam?.let { node -> + if (!node.lock.tryLock()) { + return } + node.spatial().rotateAround(yaw, pitch, target.invoke()) + node.lock.unlock() } } diff --git a/src/main/kotlin/graphics/scenery/controls/behaviours/ConfirmableClickBehaviour.kt b/src/main/kotlin/graphics/scenery/controls/behaviours/ConfirmableClickBehaviour.kt new file mode 100644 index 0000000000..a7a48cbcad --- /dev/null +++ b/src/main/kotlin/graphics/scenery/controls/behaviours/ConfirmableClickBehaviour.kt @@ -0,0 +1,33 @@ +package graphics.scenery.controls.behaviours + +import org.scijava.ui.behaviour.ClickBehaviour +import kotlin.concurrent.thread + +/** + * [ClickBehaviour] that waits [timeout] for confirmation by re-executing the behaviour. + * Executes [armedAction] on first invocation, and [confirmAction] on second invocation, if + * it happens within [timeout]. + * + * @author Ulrik Guenther + */ +class ConfirmableClickBehaviour(val armedAction: (Long) -> Any, val confirmAction: (Long) -> Any, var timeout: Long = 3000): ClickBehaviour { + /** Whether the action is armed at the moment. Action becomes disarmed after [timeout]. */ + private var armed: Boolean = false + + /** + * Action fired at position [x]/[y]. Parameters not used in VR actions. + */ + override fun click(x : Int, y : Int) { + if(!armed) { + armed = true + armedAction.invoke(timeout) + + thread { + Thread.sleep(timeout) + armed = false + } + } else { + confirmAction.invoke(timeout) + } + } +} diff --git a/src/main/kotlin/graphics/scenery/controls/behaviours/MouseDragSphere.kt b/src/main/kotlin/graphics/scenery/controls/behaviours/MouseDragSphere.kt index bbac795109..53ec77cc24 100644 --- a/src/main/kotlin/graphics/scenery/controls/behaviours/MouseDragSphere.kt +++ b/src/main/kotlin/graphics/scenery/controls/behaviours/MouseDragSphere.kt @@ -7,20 +7,27 @@ import graphics.scenery.utils.lazyLogger import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.plus import graphics.scenery.utils.extensions.times +import org.joml.Matrix4f +import org.joml.Quaternionf import org.joml.Vector3f import org.scijava.ui.behaviour.DragBehaviour +import kotlin.math.atan2 /** * Drag nodes roughly along a sphere around the camera by mouse. * Implements algorithm from https://forum.unity.com/threads/implement-a-drag-and-drop-script-with-c.130515/ - * + * @param [name] Name of the behavior + * @param [camera] The camera to use + * @param [filter] Ignore nodes for the raycast for nodes it retuns false for + * @param [rotateAroundCenter] Rotates the object around the world center instead of the camera. Defaults to false. * @author Jan Tiemann */ open class MouseDragSphere( protected val name: String, camera: () -> Camera?, protected var debugRaycast: Boolean = false, - var filter: (Node) -> Boolean + var filter: (Node) -> Boolean, + private var rotateAroundCenter: Boolean = false, ) : DragBehaviour, WithCameraDelegateBase(camera) { protected val logger by lazyLogger() @@ -33,9 +40,10 @@ open class MouseDragSphere( name: String, camera: () -> Camera?, debugRaycast: Boolean = false, - ignoredObjects: List> = listOf>(BoundingGrid::class.java) + ignoredObjects: List> = listOf>(BoundingGrid::class.java), + rotateAroundCenter: Boolean = false ) : this(name, camera, debugRaycast, { n: Node -> - !ignoredObjects.any { it.isAssignableFrom(n.javaClass) }}) + !ignoredObjects.any { it.isAssignableFrom(n.javaClass) }}, rotateAroundCenter) override fun init(x: Int, y: Int) { @@ -60,12 +68,21 @@ open class MouseDragSphere( val (rayStart, rayDir) = cam.screenPointToRay(x, y) rayDir.normalize() val newHit = rayStart + rayDir * distance - val movement = newHit - currentHit it.ifSpatial { - val newPos = position + movement / worldScale() + val newPos = if (rotateAroundCenter) { + // Calculate the rotation around (0, 0, 0) + val currentPos = position / worldScale() + val axis = currentPos.cross(movement, Vector3f()).normalize() + val angle = atan2(movement.length(), currentPos.length())//currentPos.angle(center) + val rotationQuaternion = Quaternionf().identity().rotateAxis(angle, axis) + rotationQuaternion.transform(currentPos, Vector3f()) + } else { + // Rotation around camera's center + position + movement / worldScale() + } currentNode?.spatialOrNull()?.position = newPos currentHit = newHit } diff --git a/src/main/kotlin/graphics/scenery/controls/eyetracking/PupilEyeTracker.kt b/src/main/kotlin/graphics/scenery/controls/eyetracking/PupilEyeTracker.kt index dbcae32b99..0a50d8aa13 100644 --- a/src/main/kotlin/graphics/scenery/controls/eyetracking/PupilEyeTracker.kt +++ b/src/main/kotlin/graphics/scenery/controls/eyetracking/PupilEyeTracker.kt @@ -226,7 +226,7 @@ class PupilEyeTracker(val calibrationType: CalibrationType = CalibrationType.Wor "gaze.2d.0.", "gaze.2d.1." -> { - TODO("2D gaze mapping needs a revamp") + logger.warn("2D gaze mapping needs a revamp and is not supported at the moment.") } "gaze.3d.0.", diff --git a/src/main/kotlin/graphics/scenery/numerics/Random.kt b/src/main/kotlin/graphics/scenery/numerics/Random.kt index 702c6e3e6e..fcfc957c93 100644 --- a/src/main/kotlin/graphics/scenery/numerics/Random.kt +++ b/src/main/kotlin/graphics/scenery/numerics/Random.kt @@ -20,6 +20,15 @@ class Random { /** Random number generator instance */ var rng = Random(seed) + /** + * Reseeds the PRNG with the new [seed]. If the seed is null, the value given in the + * system property `scenery.RandomSeed` will be used. + */ + @JvmStatic fun reseed(seed: Long? = null) { + this.seed = seed ?: (System.getProperty("scenery.RandomSeed")?.toLong() ?: Random.nextLong()) + rng = Random(this.seed) + } + /** * Returns a random float from the range [min]-[max]. */ diff --git a/src/main/kotlin/graphics/scenery/primitives/Atmosphere.kt b/src/main/kotlin/graphics/scenery/primitives/Atmosphere.kt new file mode 100644 index 0000000000..18390f3d16 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/primitives/Atmosphere.kt @@ -0,0 +1,135 @@ +package graphics.scenery.primitives + +import graphics.scenery.* +import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.InputHandler +import kotlinx.coroutines.* +import org.joml.Quaternionf +import org.joml.Vector3f +import org.joml.Vector4f +import org.scijava.ui.behaviour.ClickBehaviour +import java.lang.Math.toRadians +import java.time.LocalDateTime +import kotlin.collections.HashMap +import kotlin.math.* + +/** + * Implementation of a Nishita sky shader, applied to an [Icosphere] that wraps around the scene as a skybox. + * The shader code is ported from Rye Terrells [repository](https://github.com/wwwtyro/glsl-atmosphere). + * To move the sun with arrow keybinds, attach the behaviours using the [attachRotateBehaviours] function. + * @param initSunDir [Vector3f] of the sun position. Defaults to sun elevation of the current local time. + * @param emissionStrength Emission strength of the atmosphere shader. Defaults to 0.3f. + * @param latitude Latitude of the user; needed to calculate the local sun position. Defaults to 50.0, which is central Germany. + */ +open class Atmosphere(initSunDir: Vector3f? = null, emissionStrength: Float = 0.3f, var latitude: Double = 50.0) : + Icosphere(10f, 2, insideNormals = true) { + + @ShaderProperty + var sunDir: Vector3f + + private var sunDirectionManual: Boolean = false + + init { + this.name = "Atmosphere" + setMaterial(ShaderMaterial.fromClass(this::class.java)) + material { + cullingMode = Material.CullingMode.Front + depthOp = Material.DepthTest.LessEqual + emissive = Vector4f(0f, 0f, 0f, emissionStrength) + } + + // Only use time-based elevation when the formal parameter is empty + if (initSunDir == null) { + sunDir = getSunDirFromTime() + } + else { + sunDir = initSunDir + sunDirectionManual = true + } + + // Spawn a coroutine to update the sun direction + val job = CoroutineScope(Dispatchers.Default).launch { + while (!sunDirectionManual) { + sunDir = getSunDirFromTime() + // Wait 30 seconds + delay(30 * 1000) + } + } + } + + /** Turn the current local time into a sun elevation angle, encoded as cartesian [Vector3f]. + * @param localTime local time parameter, defaults to [LocalDateTime.now]. + */ + private fun getSunDirFromTime(localTime: LocalDateTime = LocalDateTime.now()): Vector3f { + val latitudeRad = toRadians(latitude) + val dayOfYear = localTime.dayOfYear.toDouble() + val declination = toRadians(-23.45 * cos(360.0 / 365.0 * (dayOfYear + 10))) + val hourAngle = toRadians((localTime.hour + localTime.minute / 60.0 - 12) * 15) + + val elevation = asin( + sin(toRadians(declination)) + * sin(latitudeRad) + + cos(declination) + * cos(latitudeRad) + * cos(hourAngle) + ) + + val azimuth = atan2( + sin(hourAngle), + cos(hourAngle) * sin(latitudeRad) - tan(declination) * cos(latitudeRad) + ) - PI / 2 + + val result = Vector3f( + cos(azimuth).toFloat(), + sin(elevation).toFloat(), + sin(azimuth).toFloat() + ) + logger.debug("Updated sun direction to {}.", result) + return result + } + + /** Move the shader sun in increments by passing a direction and optionally an increment value. + * @param arrowKey The direction to be passed as [String]. + * */ + private fun moveSun(arrowKey: String, increment: Float) { + // Indicate that the user switched to manual sun direction controls + if (!sunDirectionManual) { + sunDirectionManual = true + logger.info("Switched to manual sun direction.") + } + // Define a HashMap to map arrow key dimension strings to rotation angles and axes + val arrowKeyMappings = HashMap>() + arrowKeyMappings["UP"] = Pair(increment, Vector3f(1f, 0f, 0f)) + arrowKeyMappings["DOWN"] = Pair(-increment, Vector3f(1f, 0f, 0f)) + arrowKeyMappings["LEFT"] = Pair(increment, Vector3f(0f, 1f, 0f)) + arrowKeyMappings["RIGHT"] = Pair(-increment, Vector3f(0f, 1f, 0f)) + + val mapping = arrowKeyMappings[arrowKey] + if (mapping != null) { + val (angle, axis) = mapping + val rotation = Quaternionf().rotationAxis(toRadians(angle.toDouble()).toFloat(), axis.x, axis.y, axis.z) + sunDir.rotate(rotation) + } + } + + /** Attach Up, Down, Left, Right key mappings to the inputhandler to rotate the sun in increments. + * Keybinds are Ctrl + cursor keys for fast movement and Ctrl + Shift + cursor keys for slow movement. + * @param increment Increment value for the rotation in degrees, defaults to 20°. Slow movement is always 10% of [increment]. */ + fun attachRotateBehaviours(inputHandler: InputHandler, increment: Float = 20f) { + val incMap = mapOf( + "fast" to increment, + "slow" to increment / 10 + ) + for (speed in listOf("fast", "slow")) { + for (direction in listOf("UP", "DOWN", "LEFT", "RIGHT")) { + val clickBehaviour = ClickBehaviour { _, _ -> incMap[speed]?.let { moveSun(direction, it) } } + val bindingName = "move_sun_${direction}_$speed" + val bindingKey = if (speed == "slow") "ctrl shift $direction" else "ctrl $direction" + logger.debug("Attaching behaviour $bindingName to key $direction") + inputHandler.addBehaviour(bindingName, clickBehaviour) + inputHandler.addKeyBinding(bindingName, bindingKey) + } + } + } +} + diff --git a/src/main/kotlin/graphics/scenery/serialization/ShaderMaterialSerializer.kt b/src/main/kotlin/graphics/scenery/serialization/ShaderMaterialSerializer.kt index bc897ccab6..3305bd52b6 100644 --- a/src/main/kotlin/graphics/scenery/serialization/ShaderMaterialSerializer.kt +++ b/src/main/kotlin/graphics/scenery/serialization/ShaderMaterialSerializer.kt @@ -45,7 +45,9 @@ class ShaderMaterialSerializer: Serializer() { sm.textures = obj.textures sm.ambient = obj.ambient sm.blending = obj.blending + sm.depthOp = obj.depthOp sm.depthTest = obj.depthTest + sm.depthWrite = obj.depthWrite sm.metallic = obj.metallic sm.name = obj.name sm.roughness = obj.roughness diff --git a/src/main/kotlin/graphics/scenery/textures/Texture.kt b/src/main/kotlin/graphics/scenery/textures/Texture.kt index 748280ec05..49ac6716e9 100644 --- a/src/main/kotlin/graphics/scenery/textures/Texture.kt +++ b/src/main/kotlin/graphics/scenery/textures/Texture.kt @@ -10,6 +10,10 @@ import org.joml.Vector3i import java.io.Serializable import java.nio.ByteBuffer import java.nio.ByteOrder +import java.util.* +import java.util.concurrent.Semaphore +import java.util.concurrent.atomic.AtomicInteger +import kotlin.collections.HashSet /** @@ -39,9 +43,13 @@ open class Texture @JvmOverloads constructor( /** Linear or nearest neighbor filtering for scaling up. */ var maxFilter: FilteringMode = FilteringMode.Linear, /** Usage type */ - val usageType: HashSet = hashSetOf(UsageType.Texture) - - + val usageType: HashSet = hashSetOf(UsageType.Texture), + /** Mutex for texture data usage */ + @Transient val mutex: Semaphore = Semaphore(1), + /** Mutex for GPU upload */ + @Transient val gpuMutex: Semaphore = Semaphore(1), + /** Atomic integer to indicate GPU upload state */ + @Transient val uploaded: AtomicInteger = AtomicInteger(0), ) : Serializable, Timestamped { init { contents?.let { c -> @@ -70,6 +78,13 @@ open class Texture @JvmOverloads constructor( } } + /** + * Indicate whether a this texture is available on the GPU already. + */ + fun availableOnGPU(): Boolean { + return (uploaded.get() > 0 && (gpuMutex.availablePermits() == 1)) + } + /** * Enum class defining available texture repeat modes. */ @@ -101,9 +116,16 @@ open class Texture @JvmOverloads constructor( Linear } + /** + * Textures need to have a usage type defined. That type can be: + * [Texture] - a regular texture + * [LoadStoreImage] - a texture that can also be used as a load/storage image in a compute shader. + * [AsyncLoad] - a texture that will be asynchronously loaded by the renderer. + */ enum class UsageType { Texture, - LoadStoreImage + LoadStoreImage, + AsyncLoad } /** Companion object of [Texture], containing mainly constant defines */ @@ -121,10 +143,11 @@ open class Texture @JvmOverloads constructor( mipmap: Boolean = true, minFilter: FilteringMode = FilteringMode.Linear, maxFilter: FilteringMode = FilteringMode.Linear, - usage: HashSet = hashSetOf(UsageType.Texture) + usage: HashSet = hashSetOf(UsageType.Texture), + channels: Int = 4 ): Texture { return Texture(Vector3i(image.width, image.height, image.depth), - 4, image.type, image.contents, repeatUVW, borderColor, normalized, mipmap, usageType = usage, minFilter = minFilter, maxFilter = maxFilter) + channels, image.type, image.contents, repeatUVW, borderColor, normalized, mipmap, usageType = usage, minFilter = minFilter, maxFilter = maxFilter) } } diff --git a/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt b/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt index f0475f0cc1..c9f555cd9f 100644 --- a/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt +++ b/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt @@ -16,8 +16,9 @@ class UpdatableTexture( normalized: Boolean = true, mipmap: Boolean = true, minFilter: FilteringMode = FilteringMode.Linear, - maxFilter: FilteringMode = FilteringMode.Linear -) : Texture(dimensions, channels, type, contents, repeatUVW, borderColor, normalized, mipmap, minFilter, maxFilter) { + maxFilter: FilteringMode = FilteringMode.Linear, + usageType: HashSet = hashSetOf(UsageType.Texture), +) : Texture(dimensions, channels, type, contents, repeatUVW, borderColor, normalized, mipmap, minFilter, maxFilter, usageType) { /** Data class for encapsulating partial transfers. */ data class TextureExtents(val x: Int, val y: Int, val z: Int, val w: Int, val h: Int, val d: Int) diff --git a/src/main/kotlin/graphics/scenery/utils/DataCompressor.kt b/src/main/kotlin/graphics/scenery/utils/DataCompressor.kt new file mode 100644 index 0000000000..60c2483f85 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/utils/DataCompressor.kt @@ -0,0 +1,160 @@ +package graphics.scenery.utils + +import org.xerial.snappy.Snappy +import java.nio.ByteBuffer +import org.lwjgl.util.zstd.Zstd.* +import org.lwjgl.util.zstd.ZstdX.ZSTD_findDecompressedSize +import org.lwjgl.util.lz4.LZ4.* +import org.lwjgl.util.lz4.LZ4Frame.LZ4F_getErrorName +import org.lwjgl.util.lz4.LZ4Frame.LZ4F_isError + +/** + * Lossless compression and decompression of binary data. Currently, the [CompressionTool] + * supported are Snappy, LZ4 and ZSTD + * + * @author Aryaman Gupta + */ + +class DataCompressor (val compressionTool: CompressionTool) { + + val logger by lazyLogger() + + /** + * The tool used for compression + */ + enum class CompressionTool { + ZSTD, + LZ4, + Snappy + } + + private fun compressSnappy(compressed: ByteBuffer, uncompressed: ByteBuffer): Long { + return Snappy.compress(uncompressed, compressed).toLong() + } + + private fun decompressSnappy(decompressed: ByteBuffer, compressed: ByteBuffer): Long { + return Snappy.uncompress(compressed, decompressed).toLong() + } + + private fun compressZSTD(compressed: ByteBuffer, uncompressed: ByteBuffer, level: Int): Long { + return checkZSTD(ZSTD_compress(compressed, uncompressed, level)) + } + + private fun decompressZSTD(decompressed: ByteBuffer, compressed: ByteBuffer): Long { + checkZSTD(ZSTD_findDecompressedSize(compressed)) + return ZSTD_decompress(decompressed, compressed) + } + + private fun compressLZ4(compressed: ByteBuffer, uncompressed: ByteBuffer, level: Int): Long { + return checkLZ4F(LZ4_compress_fast(uncompressed, compressed, level).toLong()) + } + + private fun decompressLZ4(decompressed: ByteBuffer, compressed: ByteBuffer): Long { + return checkLZ4F(LZ4_decompress_safe(compressed, decompressed).toLong()) + } + + private fun checkZSTD(errorCode: Long): Long { + check(!ZSTD_isError(errorCode)) { "Zstd error: " + errorCode + " | " + ZSTD_getErrorName(errorCode) } + return errorCode + } + + private fun checkLZ4F(errorCode: Long): Long { + check(!LZ4F_isError(errorCode)) { "LZ4 error: " + errorCode + " | " + LZ4F_getErrorName(errorCode) } + return errorCode + } + + /** + * Returns the maximum size of the compressed buffer generated by the selected [CompressionTool], for + * a source buffer of size [sourceSize]. + */ + fun returnCompressBound(sourceSize: Long): Int { + return when (compressionTool) { + CompressionTool.ZSTD -> { + ZSTD_COMPRESSBOUND(sourceSize).toInt() + } + CompressionTool.LZ4 -> { + LZ4_compressBound(sourceSize.toInt()) + } + CompressionTool.Snappy -> { + Snappy.maxCompressedLength(sourceSize.toInt()) + } + } + } + + /** + * Compares buffers [uncompressed] and [decompressed] byte-by-byte. If the [decompressed] buffer + * is a result of a compression followed by decompression of the [uncompressed] buffer, this + * amounts to a verification of the compression and decompression functionalities. + * + * Returns true if the two buffers are identical, otherwise false. + */ + fun verifyDecompressed(uncompressed: ByteBuffer, decompressed: ByteBuffer): Boolean { + var verificationSuccessful = true + + val decompressedSize = decompressed.remaining().toLong() + if(decompressedSize != uncompressed.remaining().toLong()) { + verificationSuccessful = false + logger.info( + "Decompressed size {} != uncompressed size {}", + decompressedSize, + uncompressed.remaining() + ) + } + + for (i in 0 until uncompressed.remaining()) { + if(decompressed[i] != uncompressed[i]) { + verificationSuccessful = false + logger.debug("Decompressed != uncompressed at: $i") + } + } + + return verificationSuccessful + } + + /** + * Compress buffer using tool [compressionTool] + * + * @param[uncompressed] the buffer to be compressed + * @param[compressed] the buffer the compressed result is stored in + * @param[level] for ZSTD and LZ4, the desired level of compression. For LZ4, higher values lead to faster, but less + * compression, while for ZSTD, the opposite is true. Optional parameter. + * + * Returns the length (in bytes) of the compressed buffer + */ + fun compress(compressed: ByteBuffer, uncompressed: ByteBuffer, level: Int? = null): Long { + val compressionLevel = level + ?: 0 + + return when (compressionTool) { + CompressionTool.ZSTD -> { + compressZSTD(compressed, uncompressed, compressionLevel) + } + CompressionTool.LZ4 -> { + compressLZ4(compressed, uncompressed, compressionLevel) + } + CompressionTool.Snappy -> { + compressSnappy(compressed, uncompressed) + } + } + } + + /** + * Decompress buffer compressed using tool [compressionTool] + * + * @param[compressed] the buffer to be compressed + * @param[decompressed] the buffer the decompressed result is stored in + */ + fun decompress(decompressed: ByteBuffer, compressed: ByteBuffer): Long { + return when (compressionTool) { + CompressionTool.ZSTD -> { + decompressZSTD(decompressed, compressed) + } + CompressionTool.LZ4 -> { + decompressLZ4(decompressed, compressed) + } + CompressionTool.Snappy -> { + decompressSnappy(decompressed, compressed) + } + } + } +} diff --git a/src/main/kotlin/graphics/scenery/utils/ExtractsNatives.kt b/src/main/kotlin/graphics/scenery/utils/ExtractsNatives.kt index c30786bcf0..7ead243914 100644 --- a/src/main/kotlin/graphics/scenery/utils/ExtractsNatives.kt +++ b/src/main/kotlin/graphics/scenery/utils/ExtractsNatives.kt @@ -85,135 +85,67 @@ interface ExtractsNatives { } } - } + /** + * Utility function to extract native libraries from the classpath, and store them in a + * temporary directory. + * + * @param[paths] A list of JAR paths to extract natives from. + * @param[load] Whether or not to directly load the extracted libraries. + */ + fun extractLibrariesFromClasspath(paths: List, load: Boolean = false): String { + val logger = LoggerFactory.getLogger(Companion::class.java.simpleName) - /** - * Cleans old temporary native libraries, e.g. all directories in the temporary directory, - * which have "scenery-natives-tmp" in their name, and do not have a lock file present. - */ - fun cleanTempFiles() { - File(System.getProperty("java.io.tmpdir")).listFiles().forEach { file -> - if (file.isDirectory && file.name.contains("scenery-natives-tmp")) { - val lock = File(file, ".lock") - - // delete the temporary directory only if the lock does not exist - if (!lock.exists()) { - file.deleteRecursively() - } + if(paths.isEmpty()) { + throw IllegalStateException("Empty path list handed to extractLibrariesFromClasspath()") } - } - } - /** - * Utility function to extract native libraries from a given JAR, store them in a - * temporary directory and modify the JRE's library path such that it can find - * these libraries. - * - * @param[paths] A list of JAR paths to extract natives from. - * @param[replace] Whether or not the java.library.path should be replaced. - */ - fun extractLibrariesFromJar(paths: List, replace: Boolean = false, load: Boolean = false): String { - // FIXME: Kotlin bug, revert to LazyLogger as soon as https://youtrack.jetbrains.com/issue/KT-19690 is fixed. - // val logger by LazyLogger() - val logger = LoggerFactory.getLogger(this.javaClass.simpleName) - - val tmpDir = Files.createTempDirectory("scenery-natives-tmp").toFile() - val lock = File(tmpDir, ".lock") - lock.createNewFile() - lock.deleteOnExit() - - cleanTempFiles() - val files = ArrayList() - - val nativeLibraryExtensions = hashMapOf( - Platform.WINDOWS to listOf("dll"), - Platform.LINUX to listOf("so"), - Platform.MACOS to listOf("dylib", "jnilib")) - - logger.debug("Got back ${paths.joinToString(", ")}") - paths.filter { it.lowercase().endsWith("jar") }.forEach { - logger.debug("Extracting $it...") - - val jar = JarFile(it) - val enumEntries = jar.entries() - - while (enumEntries.hasMoreElements()) { - val file = enumEntries.nextElement() - if(file.getName().substringAfterLast(".") !in nativeLibraryExtensions[getPlatform()]!!) { - continue - } - files.add(tmpDir.absolutePath + File.separator + file.getName()) - val f = File(files.last()) - - // create directory, if needed - if (file.isDirectory()) { - f.mkdir() - continue - } + val tmpDir = Files.createTempDirectory("scenery-natives-tmp").toFile() + val lock = File(tmpDir, ".lock") + lock.createNewFile() + lock.deleteOnExit() - val ins = jar.getInputStream(file) - val baos = ByteArrayOutputStream() - val fos = FileOutputStream(f) + cleanTempFiles() - val buffer = ByteArray(1024) - var len: Int = ins.read(buffer) + paths.forEach { nativeLibrary -> + val f = this::class.java.classLoader.getResourceAsStream(nativeLibrary) - while (len > -1) { - baos.write(buffer, 0, len) - len = ins.read(buffer) + if(f == null) { + logger.warn("Could not find native library $nativeLibrary in classpath.") + return@forEach } - baos.flush() - fos.write(baos.toByteArray()) - - if(getPlatform() == Platform.MACOS && file.name.substringAfterLast(".") == "jnilib") { - logger.debug("macOS: Making dylib copy of jnilib file for compatibility") - try { - f.copyTo(File(tmpDir.absolutePath + File.separator + file.name.substringBeforeLast(".") + ".dylib"), false) - } catch (e: IOException) { - logger.warn("Failed to create copy of ${file.name} to ${file.name.substringBeforeLast(".")}.dylib") - } + val out = tmpDir.resolve(nativeLibrary) + if(!out.exists()) { + val outputStream = out.outputStream() + f.copyTo(outputStream) + outputStream.close() + logger.info("Extracted native library $nativeLibrary to ${out.absolutePath}") } - fos.close() - baos.close() - ins.close() - } - } - - if(load) { - files.forEach { lib -> - logger.debug("Loading native library $lib") - System.load(lib) + if(load) { + logger.info("Loading native library from ${out.absolutePath}") + System.load(out.absolutePath) + } } - } - return tmpDir.absolutePath - } - - /** - * Utility function to search the current class path for JARs with native libraries - * - * @param[searchName] The string to match the JAR's name against - * @param[hint] A file name to look for, for the ImageJ classpath hack - * @return A list of JARs matching [searchName] - */ - fun getNativeJars(searchName: String, hint: String = ""): List { - val res = Thread.currentThread().contextClassLoader.getResource(hint) - - if (res == null) { - LoggerFactory.getLogger(this.javaClass.simpleName).error("Could not find JAR matching \"" + searchName + "\" with native libraries (${getPlatform()}, $hint).") - return listOf() + return tmpDir.absolutePath } - var jar = res.path - var pathOffset = 5 - - if (getPlatform() == Platform.WINDOWS) { - pathOffset = 6 + /** + * Cleans old temporary native libraries, e.g. all directories in the temporary directory, + * which have "scenery-natives-tmp" in their name, and do not have a lock file present. + */ + private fun cleanTempFiles() { + File(System.getProperty("java.io.tmpdir")).listFiles().forEach { file -> + if (file.isDirectory && file.name.contains("scenery-natives-tmp")) { + val lock = File(file, ".lock") + + // delete the temporary directory only if the lock does not exist + if (!lock.exists()) { + file.deleteRecursively() + } + } + } } - - jar = jar.substring(jar.indexOf("file:/") + pathOffset).substringBeforeLast("!") - return jar.split(File.pathSeparator) } } diff --git a/src/main/kotlin/graphics/scenery/utils/ParallelHelpers.kt b/src/main/kotlin/graphics/scenery/utils/ParallelHelpers.kt index a1f2f2ac2c..24c38139e7 100644 --- a/src/main/kotlin/graphics/scenery/utils/ParallelHelpers.kt +++ b/src/main/kotlin/graphics/scenery/utils/ParallelHelpers.kt @@ -9,6 +9,8 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger import kotlin.concurrent.thread +import kotlin.time.Duration +import kotlin.time.Duration.Companion.nanoseconds /** * Maps the function [f] asynchronously on [this], returning the resultant list. @@ -114,3 +116,25 @@ fun HashMap.forEachParallel(maxThreads: Int = 5, action: ((K, V) -> } } } + +/** + * Launches a [Job] given by [action] that will be executed periodically, with a minimum delay + * of [every] between individual launches. [every] can be an arbitrary [Duration] bigger than 0ns. + */ +fun CoroutineScope.launchPeriodicAsync( + every: Duration, + action: () -> Boolean +) = this.async { + if (every > 0.nanoseconds) { + while (isActive) { + val result = action() + if(result) { + break + } + delay(every) + } + } else { + action() + } +} + diff --git a/src/main/kotlin/graphics/scenery/utils/SceneryJPanel.kt b/src/main/kotlin/graphics/scenery/utils/SceneryJPanel.kt index 10310d6016..a49f5f5c22 100644 --- a/src/main/kotlin/graphics/scenery/utils/SceneryJPanel.kt +++ b/src/main/kotlin/graphics/scenery/utils/SceneryJPanel.kt @@ -13,7 +13,7 @@ import javax.swing.JPanel * * @author Ulrik Guenther */ -class SceneryJPanel : JPanel(), SceneryPanel { +class SceneryJPanel(val owned: Boolean = false) : JPanel(), SceneryPanel { /** Refresh rate. */ override var refreshRate: Int = 60 diff --git a/src/main/kotlin/graphics/scenery/utils/Statistics.kt b/src/main/kotlin/graphics/scenery/utils/Statistics.kt index ea1ecf46c5..2badfc5107 100644 --- a/src/main/kotlin/graphics/scenery/utils/Statistics.kt +++ b/src/main/kotlin/graphics/scenery/utils/Statistics.kt @@ -85,6 +85,14 @@ class Statistics(override var hub: Hub?) : Hubable { } } + /** + * Remove stat [name] from [stats]. Can be used, e.g., to reset + * the stat. + */ + fun clear(name: String) { + stats.remove(name) + } + /** * Adds a new datum to the statistic about [name] with [value]. * Accepts all types of numbers. diff --git a/src/main/kotlin/graphics/scenery/utils/SystemHelpers.kt b/src/main/kotlin/graphics/scenery/utils/SystemHelpers.kt index dd346049f5..1d67718d56 100644 --- a/src/main/kotlin/graphics/scenery/utils/SystemHelpers.kt +++ b/src/main/kotlin/graphics/scenery/utils/SystemHelpers.kt @@ -243,12 +243,11 @@ class SystemHelpers { */ fun dumpToFile(buf: ByteBuffer, filename: String) { try { + val view = buf.duplicate().order(ByteOrder.LITTLE_ENDIAN) val file = File(filename) val channel = FileOutputStream(file, false).channel - channel.write(buf) + channel.write(view) channel.close() - - buf.flip() } catch (e: Exception) { logger.error("Unable to dump byte buffer to $filename") e.printStackTrace() diff --git a/src/main/kotlin/graphics/scenery/utils/VideoDecoder.kt b/src/main/kotlin/graphics/scenery/utils/VideoDecoder.kt index 43c4d56521..5dfb86bb23 100644 --- a/src/main/kotlin/graphics/scenery/utils/VideoDecoder.kt +++ b/src/main/kotlin/graphics/scenery/utils/VideoDecoder.kt @@ -2,6 +2,7 @@ package graphics.scenery.utils import org.bytedeco.ffmpeg.avcodec.AVPacket import org.bytedeco.ffmpeg.avformat.AVFormatContext +import org.bytedeco.ffmpeg.avutil.AVDictionary import org.bytedeco.ffmpeg.avutil.AVFrame import org.bytedeco.ffmpeg.global.avcodec.* import org.bytedeco.ffmpeg.global.avformat.* @@ -56,7 +57,12 @@ class VideoDecoder(val filename: String) { val videoPath = filename - ret = avformat_open_input(formatContext, videoPath, null, null) + // We need an options dictionary here, so the sdp files produced + // by [VideoEncoder] are ingestible again + val d = AVDictionary() + av_dict_set(d, "protocol_whitelist", "file,udp,rtp", 0); + + ret = avformat_open_input(formatContext, videoPath, null, d) if (ret < 0) { eVal = ret logger.error("Open video file $videoPath failed. Error code: $eVal") @@ -92,10 +98,8 @@ class VideoDecoder(val filename: String) { return@thread } else { logger.debug( - "Video stream %d with resolution %dx%d\n", vidStreamIdx, - formatContext.streams(i).codecpar().width(), - formatContext.streams(i).codecpar().height() - ) + "Video stream $vidStreamIdx with resolution resolution ${formatContext.streams(i).codecpar().width()}x" + + "${formatContext.streams(i).codecpar().height()}\n") } ret = avcodec_parameters_to_context(codecCtx, formatContext.streams(vidStreamIdx).codecpar()) @@ -107,6 +111,7 @@ class VideoDecoder(val filename: String) { } val codec = avcodec_find_decoder(codecCtx.codec_id()) +// TODO: for h264 hardware decoding on Nvidia: val codec = avcodec_find_decoder_by_name("h264_cuvid") if (codec == null) { logger.error("Unsupported codec for video file") error = true @@ -174,6 +179,12 @@ class VideoDecoder(val filename: String) { } } + /** + * Decodes and returns the next frame in the video stream. + * + * @return The decoded image as a [ByteArray] + */ + fun decodeFrame(): ByteArray? { var finalize = false @@ -229,6 +240,27 @@ class VideoDecoder(val filename: String) { return image } + /** + * Decodes the video stream frame-by-frame until the end and applies the lambda [f] on each decoded image. + * + * @param[f] The lambda to be applied to each decoded frame taking the decoded image ([ByteArray]), + * frame width (Int), height (Int) and frame number (Int) as parameters. + */ + fun decodeFrameByFrame(f: (ByteArray, Int, Int, Int) -> Unit) { + var decodedFrameCount = 0 + + while (nextFrameExists) { + val image = decodeFrame() + if(image != null) { // image can be null, e.g. when the decoder encounters invalid information between frames + decodedFrameCount++ + f.invoke(image, videoWidth, videoHeight, decodedFrameCount) + } + } + + logger.debug("closing video decoder") + close() + } + private fun getImage(pFrame: AVFrame, width: Int, height: Int) : ByteArray { val image = ByteArray(width * height * 4) val data = pFrame.data(0) @@ -246,4 +278,11 @@ class VideoDecoder(val filename: String) { return String(buffer, 0, buffer.indexOfFirst { it == 0.toByte() }) } -} \ No newline at end of file + + /** + * Closes the video decoder and frees up allocated resources. + */ + fun close () { + avformat_close_input(formatContext) + } +} diff --git a/src/main/kotlin/graphics/scenery/utils/extensions/CameraUtils.kt b/src/main/kotlin/graphics/scenery/utils/extensions/CameraUtils.kt new file mode 100644 index 0000000000..1cd6ece3e9 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/utils/extensions/CameraUtils.kt @@ -0,0 +1,20 @@ +package graphics.scenery.utils.extensions + +import org.joml.Matrix4f + +private val vulkanProjectionFix = + Matrix4f( + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.5f, 0.0f, + 0.0f, 0.0f, 0.5f, 1.0f) + +/** + * Converts a camera projection matrix from OpenGL to Vulkan coordinate system conventions. + */ +fun Matrix4f.applyVulkanCoordinateSystem(): Matrix4f { + val m = Matrix4f(vulkanProjectionFix) + m.mul(this) + + return m +} diff --git a/src/main/kotlin/graphics/scenery/utils/extensions/TextureUtils.kt b/src/main/kotlin/graphics/scenery/utils/extensions/TextureUtils.kt new file mode 100644 index 0000000000..2f9935865b --- /dev/null +++ b/src/main/kotlin/graphics/scenery/utils/extensions/TextureUtils.kt @@ -0,0 +1,38 @@ +package graphics.scenery.utils.extensions + +import graphics.scenery.backends.vulkan.VulkanTexture +import graphics.scenery.textures.Texture +import graphics.scenery.utils.lazyLogger +import org.jetbrains.annotations.ApiStatus.Experimental + + +/** + * Fetches this [Texture] from the GPU. This replaced the texture's + * contents with whatever is fetched from the GPU and returns true. + * If the texture is not known to the renderer, or no storage buffer + * is available, it'll return false. + */ +@Experimental +fun Texture.fetchFromGPU(): Boolean { + val logger by lazyLogger() + + val buffer = this.contents + if(buffer == null) { + logger.error("Texture copy from GPU requested, but no storage available.") + return false + } + + val ref = VulkanTexture.getReference(this) + + if(ref != null) { + val start = System.nanoTime() + this.contents = ref.copyTo(buffer, true) + val end = System.nanoTime() + logger.info("The request textures of size ${this.contents?.remaining()?.toFloat()?.div((1024f*1024f))} took: ${(end.toDouble()-start.toDouble())/1000000.0}") + } else { + logger.error("In fetchFromGPU: Texture not accessible") + return false + } + + return true +} diff --git a/src/main/kotlin/graphics/scenery/utils/extensions/VolumeUtils.kt b/src/main/kotlin/graphics/scenery/utils/extensions/VolumeUtils.kt new file mode 100644 index 0000000000..112c5a1911 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/utils/extensions/VolumeUtils.kt @@ -0,0 +1,25 @@ +package graphics.scenery.utils.extensions + +import graphics.scenery.RichNode +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.Volume.Companion.fromPathRawSplit +import org.joml.Vector3f + +/** + * Positions [volumes] back-to-back without gaps, using their pixel-to-world ratio. Can, e.g., be used + * with [fromPathRawSplit] to load volume files greater than 2 GiB into sliced partitions and place + * the partitions back-to-back, emulating a single large volume in the scene. + */ +fun RichNode.positionVolumeSlices(volumes: List) { + val pixelToWorld = volumes.first().pixelToWorldRatio + + var sliceIndex = 0 + volumes.forEach { volume -> + val currentSlices = volume.getDimensions().z + logger.debug("Volume partition with z slices: $currentSlices") + volume.pixelToWorldRatio = pixelToWorld + + volume.spatial().position = Vector3f(0f, 0f, 1.0f * (sliceIndex) * pixelToWorld) + sliceIndex += currentSlices + } +} \ No newline at end of file diff --git a/src/main/kotlin/graphics/scenery/volumes/BufferedVolume.kt b/src/main/kotlin/graphics/scenery/volumes/BufferedVolume.kt index 6e5e8cc33e..71a2d508ca 100644 --- a/src/main/kotlin/graphics/scenery/volumes/BufferedVolume.kt +++ b/src/main/kotlin/graphics/scenery/volumes/BufferedVolume.kt @@ -4,23 +4,23 @@ import bdv.tools.transformation.TransformedSource import bvv.core.VolumeViewerOptions import graphics.scenery.Hub import graphics.scenery.OrientedBoundingBox +import graphics.scenery.SceneryElement +import graphics.scenery.backends.Renderer import graphics.scenery.utils.extensions.minus import graphics.scenery.utils.extensions.plus import graphics.scenery.utils.extensions.times -import net.imglib2.type.numeric.NumericType import net.imglib2.type.numeric.integer.* import net.imglib2.type.numeric.real.DoubleType import net.imglib2.type.numeric.real.FloatType +import org.jfree.data.statistics.SimpleHistogramBin +import org.jfree.data.statistics.SimpleHistogramDataset import org.joml.Vector3f import org.joml.Vector3i import org.lwjgl.system.MemoryUtil import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.concurrent.CopyOnWriteArrayList -import kotlin.math.floor -import kotlin.math.max -import kotlin.math.min -import kotlin.math.roundToInt +import kotlin.math.* /** * Convenience class to handle buffer-based volumes. Data descriptor is stored in [ds], similar @@ -292,4 +292,40 @@ class BufferedVolume(val ds: VolumeDataSource.RAISource<*>, options: VolumeViewe val source = ((ds.sources.first().spimSource as? TransformedSource)?.wrappedSource as? BufferSource) ?: throw IllegalStateException("No source found") return Vector3i(source.width, source.height, source.depth) } + + /** + * Generates a histogram using GPU acceleration via [VolumeHistogramComputeNode]. + */ + override fun generateHistogram(volumeHistogramData: SimpleHistogramDataset): Int? { + + val volumeHistogram = VolumeHistogramComputeNode.generateHistogram( + this, + timepoints?.get(currentTimepoint)!!.contents, + getScene() ?: return null + ) + + val histogram = volumeHistogram.fetchHistogram( + getScene()!!, volumeManager.hub!!.get( + SceneryElement.Renderer + )!! + ) + + val displayRange = abs(maxDisplayRange - minDisplayRange) + val binSize = displayRange / volumeHistogram.numBins + val minDisplayRange = minDisplayRange.toDouble() + + var max = 0 + histogram.forEachIndexed { index, value -> + val bin = SimpleHistogramBin( + minDisplayRange + index * binSize, + minDisplayRange + (index + 1) * binSize, + true, + false + ) + bin.itemCount = value + volumeHistogramData.addBin(bin) + max = max(max, value) + } + return max + } } diff --git a/src/main/kotlin/graphics/scenery/volumes/Colormap.kt b/src/main/kotlin/graphics/scenery/volumes/Colormap.kt index 772be45bac..0e60b3fc6b 100644 --- a/src/main/kotlin/graphics/scenery/volumes/Colormap.kt +++ b/src/main/kotlin/graphics/scenery/volumes/Colormap.kt @@ -6,11 +6,13 @@ import net.imagej.lut.LUTService import net.imglib2.display.ColorTable import org.joml.Vector4f import org.scijava.plugin.Parameter +import java.awt.image.BufferedImage +import java.io.File import java.io.FileNotFoundException import java.io.IOException import java.io.InputStream import java.nio.ByteBuffer -import java.util.* +import javax.imageio.ImageIO import kotlin.math.ceil import kotlin.math.floor import kotlin.math.roundToInt @@ -20,9 +22,9 @@ import kotlin.math.roundToInt */ class Colormap(val buffer: ByteBuffer, val width: Int, val height: Int) { - private constructor() : this(ByteBuffer.allocate(0), 0, 0) { - - } + // This needs to stay, Kryo needs it for (de)serialisation + @Suppress("unused") + private constructor() : this(ByteBuffer.allocate(0), 0, 0) /** * Returns the value of the colormap, sampled at [position]. @@ -128,6 +130,22 @@ class Colormap(val buffer: ByteBuffer, val width: Int, val height: Int) { return fromBuffer(byteBuffer, width, copies) } + /** + * Creates a color map from a png file. + */ + fun fromPNGFile(file: File): Colormap { + var img: BufferedImage? = null + try { + img = ImageIO.read(file) + } catch (_: IllegalArgumentException){ + logger.error("Could not find file ${file.path}") + } catch (e: IOException){ + logger.error(e.toString()) + } + if (img == null) throw IllegalArgumentException("Could not open png file $file") + return fromBuffer(Image.bufferedImageToRGBABuffer(img),img.width, img.height) + } + /** * Tries to load a colormap from a file. Available colormaps can be queried with [list]. */ @@ -155,7 +173,7 @@ class Colormap(val buffer: ByteBuffer, val width: Int, val height: Int) { */ @JvmStatic fun list(): List { // FIXME: Hardcoded for the moment, not nice. - val list = arrayListOf("grays", "hot", "jet", "plasma", "viridis") + val list = arrayListOf("grays", "hot", "jet", "plasma", "viridis", "red-blue", "rb-darker") lutService?.findLUTs()?.keys?.forEach { list.add(it) } return list diff --git a/src/main/kotlin/graphics/scenery/volumes/ColormapPanel.kt b/src/main/kotlin/graphics/scenery/volumes/ColormapPanel.kt new file mode 100644 index 0000000000..03e760ba3b --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/ColormapPanel.kt @@ -0,0 +1,411 @@ +package graphics.scenery.volumes + +import graphics.scenery.utils.Image +import net.miginfocom.swing.MigLayout +import org.apache.commons.io.FilenameUtils +import org.joml.Math.clamp +import java.awt.* +import java.awt.event.MouseEvent +import java.awt.event.MouseListener +import java.awt.event.MouseMotionListener +import java.awt.image.BufferedImage +import java.io.File +import java.io.IOException +import java.nio.ByteBuffer +import javax.imageio.ImageIO +import javax.swing.* +import javax.swing.border.MatteBorder +import javax.swing.border.TitledBorder +import javax.swing.event.PopupMenuEvent +import javax.swing.event.PopupMenuListener +import javax.swing.filechooser.FileFilter +import kotlin.math.absoluteValue + + +/** + * A JPanel to display everything related to setting and editing the color map of a volume. + * + * @author Jan Tiemann + * @author Aryaman Gupta + */ +class ColormapPanel(val target:Volume?): JPanel() { + private val colorMapEditor = ColormapEditor(target) + private val loadedColormaps = HashMap() + + init { + layout = MigLayout("insets 0", "[][][][]") + val title = object: TitledBorder(MatteBorder(1, 0, 0, 0, Color.GRAY), "Color Map") { + val customInsets = Insets(25, 0, 25, 0) + override fun getBorderInsets(c: Component?): Insets { + return customInsets + } + + override fun getBorderInsets(c: Component?, insets: Insets?): Insets { + return customInsets + } + } + border = title + + // color editor + this.add(colorMapEditor, "spanx, growx, wrap") + + // color map drop down + val list = Colormap.list() + val box = JComboBox() + val selectAColorMapString = "Select ..." // makes codacy stop complaining + box.addItem(selectAColorMapString) + + for (s in list) { + box.addItem(s) + } + + if (target != null) { + box.selectedItem = selectAColorMapString + add(box, "grow") + } + + box.addActionListener { + val item: String = box.selectedItem as String + var colormap: Colormap? = null + + if (target != null && item != selectAColorMapString) { + // try to load from already-seen files first + colormap = loadedColormaps[item] ?: Colormap.get(item) + target.colormap = colormap + this.repaint() + } + + colormap?.let { colorMapEditor.loadColormap(it) } + } + + val fc = JFileChooser() + fc.addChoosableFileFilter(PNGFileFilter()) + fc.isAcceptAllFileFilterUsed = false + + val colormapMenu = JPopupMenu() + colormapMenu.add(JMenuItem("Load colormap ...").also { + it.toolTipText = "Load a new colormap from a file" + it.addActionListener { + val returnVal: Int = fc.showOpenDialog(this) + if (returnVal == JFileChooser.APPROVE_OPTION) { + val newColormap = Colormap.fromPNGFile(fc.selectedFile) + val filename = fc.selectedFile.nameWithoutExtension + @Suppress("UNCHECKED_CAST") + val currentItems = box.items() as List + + val colormapName = if(filename in currentItems) { + println(currentItems.joinToString(",")) + val index = currentItems.count { n -> n.startsWith(filename) } + 1 + "$filename ($index)" + } else { + filename + } + + loadedColormaps[colormapName] = newColormap + colorMapEditor.loadColormap(newColormap) + box.addItem(colormapName) + box.selectedItem = colormapName + } + } + + }) + colormapMenu.add(JMenuItem("Save colormap ...").also { + it.toolTipText = "Save the current colormap to a file" + it.addActionListener { + val option = fc.showSaveDialog(this) + if (option == JFileChooser.APPROVE_OPTION) { + saveToFile (fc.selectedFile) + } + } + }) + + val colormapMenuButton = JToggleButton("").also { button -> + button.icon = ImageIcon(ImageIcon(ImageIO.read(this::class.java.getResource("/graphics/scenery/ui/gear.png"))).image.getScaledInstance(16, 16, + java.awt.Image.SCALE_SMOOTH + )) + button.toolTipText = "Load a new transfer function and display range" + button.addActionListener { + if(button.isSelected) { + colormapMenu.show(button, 0, button.height) + } else { + colormapMenu.isVisible = false + } + } + + add(button, "skip 2, al right, push") + } + + colormapMenu.addPopupMenuListener(object: PopupMenuListener { + override fun popupMenuWillBecomeVisible(e: PopupMenuEvent?) { + /* not used */ + } + + override fun popupMenuWillBecomeInvisible(e: PopupMenuEvent?) { + colormapMenuButton.isSelected = false + } + + override fun popupMenuCanceled(e: PopupMenuEvent?) { + colormapMenuButton.isSelected = false + } + + }) + } + + private fun JComboBox<*>.items(): List { + val items = ArrayList() + for(i in 0 until this.itemCount) { + items.add(this.getItemAt(i)) + } + return items + } + + /** + * A filter to only select pngs. + */ + private class PNGFileFilter: FileFilter() + { + override fun accept(f: File): Boolean { + if (f.isDirectory()) { + return false + } + if (f.extension != "png") return false + return true + } + + override fun getDescription(): String { + return "png" + } + } + + /** + * Save a color map to file as a png. + */ + fun saveToFile(file: File){ + var fileTemp = file + if (FilenameUtils.getExtension(file.name).equals("png", ignoreCase = true)) { + // filename is OK as-is + } else { + fileTemp = File("$file.png") // append .png if "foo.jpg.png" is OK + } + + try { + ImageIO.write(colorMapEditor.toImage(), "png", fileTemp) + } catch (ioe: IOException) { + ioe.printStackTrace() + } + } + /** + * A GUI element to allow users to visually create or modify a color map + */ + class ColormapEditor(var target:Volume? = null) : JPanel() { + + private var colorPoints = listOf( + ColorPoint(0.0f, Color(0f, 0f, 0f)), + ColorPoint(0.5f, Color(0f, 0.5f, 0f)), + ColorPoint(1f, Color(0f, 1f, 0f)) + ) + + private var hoveredOver: ColorPoint? = null + private var dragging: ColorPoint? = null + private var dragged = false + + init { + this.layout = MigLayout() + this.preferredSize = Dimension(1000, 40) + + target?.let { loadColormap(it.colormap) } + + this.addMouseListener(object : MouseListener { + override fun mouseClicked(e: MouseEvent) { + val point = pointAtMouse(e) + when { + SwingUtilities.isLeftMouseButton(e) && point != null && !dragged -> { + point.color = + JColorChooser.showDialog(null, "Choose a color for point", point.color) ?: point.color + repaintAndReassign() + } + + SwingUtilities.isRightMouseButton(e) && point != null -> { + if (0f < point.position && point.position < 1.0f) { + // dont delete first and last point + colorPoints = colorPoints - point + } + } + + point == null -> { + val pos = e.x/ width.toFloat() + val pointList = colorPoints.sortedBy { it.position } + + var red = 0f + var green = 0f + var blue = 0f + + for(i in pointList.indices) { + if(pointList[i].position <= pos && pointList[i+1].position >= pos) { + val interpolationFactor = (pos - pointList[i].position) / (pointList[i+1].position - pointList[i].position) + + red = pointList[i].color.red.toFloat()/255.0f + interpolationFactor * (pointList[i+1].color.red.toFloat()/255.0f - pointList[i].color.red.toFloat()/255.0f) + green = pointList[i].color.green.toFloat()/255.0f + interpolationFactor * (pointList[i+1].color.green.toFloat()/255.0f - pointList[i].color.green.toFloat()/255.0f) + blue = pointList[i].color.blue.toFloat()/255.0f + interpolationFactor * (pointList[i+1].color.blue.toFloat()/255.0f - pointList[i].color.blue.toFloat()/255.0f) + break + } + } + + val color = Color(red, green, blue) + colorPoints += ColorPoint((e.x / width.toFloat()), color) + } + } + repaintAndReassign() + } + + override fun mousePressed(e: MouseEvent) { + val cp = pointAtMouse(e) + cp?.let { + if (0f < cp.position && cp.position < 1.0f) { + // dont move first and last point + dragging = cp + } + } + dragged = false + } + + override fun mouseReleased(e: MouseEvent?) { + dragging = null + } + + override fun mouseEntered(e: MouseEvent?) {/*noop*/} + override fun mouseExited(e: MouseEvent?) {/*noop*/} + }) + + this.addMouseMotionListener(object : MouseMotionListener { + override fun mouseDragged(e: MouseEvent) { + dragging?.position = clamp(0.05f, 0.95f, e.x / width.toFloat()) + dragged = true + repaintAndReassign() + } + + override fun mouseMoved(e: MouseEvent) { + val temp = hoveredOver + hoveredOver = pointAtMouse(e) // height is the radius of the color point sphere + if (temp != hoveredOver) { + repaintAndReassign() + } + } + }) + } + + private fun repaintAndReassign() { + repaint() + + if(width > 0 && height > 0) { + target?.colormap = Colormap.fromBuffer(toBuffer(), width, height) + } + } + + internal fun toImage(): BufferedImage { + val rec: Rectangle = this.bounds + val bufferedImage = BufferedImage(rec.width, rec.height, BufferedImage.TYPE_INT_ARGB) + paintBackgroundGradient(colorPoints.sortedBy { it.position }, bufferedImage.graphics as Graphics2D) + return bufferedImage + } + + private fun toBuffer(): ByteBuffer { + return Image.bufferedImageToRGBABuffer(toImage()) + } + + private fun pointAtMouse(e: MouseEvent) = + colorPoints.firstOrNull { (e.x - (width * it.position)).absoluteValue < height / 2 } + + override fun paintComponent(g: Graphics) { + super.paintComponent(g) + val g2d = g as Graphics2D + g2d.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY) + val w = width + val h = height + val pointList = colorPoints.sortedBy { it.position } + + // background Gradient + paintBackgroundGradient(pointList, g2d) + + // color point markers + val relativeSize = 0.25f //relative to height + val absoluteSize = (relativeSize * h).toInt() + + pointList.forEach { + val margin = 0.35f + + val innerSize = (absoluteSize * (1.0f - margin)).toInt() + val p1x = w * it.position + val colorHSB = floatArrayOf(0.0f, 0.0f, 0.0f) + Color.RGBtoHSB(it.color.red, it.color.green, it.color.blue, colorHSB) + + val backgroundDark = colorHSB[2] < 0.5f + + val outlineColor = if(backgroundDark) { + Color.BLACK + } else { + Color.WHITE + } + + // This draws a triangle below the gradient bar to indicate control points + g2d.paint = outlineColor + g2d.drawPolygon(intArrayOf(p1x.toInt(), (p1x - innerSize).toInt(), (p1x + innerSize).toInt()), + intArrayOf(h-10, h-1, h-1), 3) + g2d.paint = it.color + g2d.fillPolygon(intArrayOf(p1x.toInt(), (p1x - innerSize-1).toInt(), (p1x + innerSize+1).toInt()), + intArrayOf(h-10-1, h, h), 3) + } + } + + private fun paintBackgroundGradient( + pointList: List, + g2d: Graphics2D + ) { + val w = width + val h = height + for (i in 0 until pointList.size - 1) { + val p1 = pointList[i] + val p2 = pointList[i + 1] + val p1x = w * p1.position + val p2x = w * p2.position + val gp = GradientPaint(p1x, 0f, p1.color, p2x, 0f, p2.color) + + g2d.paint = gp + g2d.fillRect(p1x.toInt(), 0, p2x.toInt(), h-10) + } + } + + private class ColorPoint(var position: Float, var color: Color) + + /** + * Loads a [Colormap] into the editor. + */ + fun loadColormap(colormap: Colormap) { + val width = colormap.width + val numPoints = 10.coerceAtMost(width) + + val sampleDistance = 1.0f/(numPoints - 2) + + var colorPointsList: List = listOf() + + //first sample + var sample = colormap.sample(0f) + colorPointsList = colorPointsList + ColorPoint(0f, Color(sample.x, sample.y, sample.z, sample.w)) + + //middle samples + for(i in 1..numPoints-2) { + sample = colormap.sample(i*sampleDistance) + colorPointsList = colorPointsList + ColorPoint(i*sampleDistance, Color(sample.x, sample.y, sample.z, sample.w)) + } + + //last sample + sample = colormap.sample(1f) + colorPointsList = colorPointsList + ColorPoint(1f, Color(sample.x, sample.y, sample.z, sample.w)) + + colorPoints = colorPointsList + + repaintAndReassign() + } + } + +} diff --git a/src/main/kotlin/graphics/scenery/volumes/DisplayRangeEditor.kt b/src/main/kotlin/graphics/scenery/volumes/DisplayRangeEditor.kt new file mode 100644 index 0000000000..3b161ad134 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/DisplayRangeEditor.kt @@ -0,0 +1,122 @@ +package graphics.scenery.volumes + +import graphics.scenery.ui.RangeSlider +import net.miginfocom.swing.MigLayout +import java.awt.Color +import java.awt.Component +import java.awt.Insets +import javax.swing.* +import javax.swing.border.MatteBorder +import javax.swing.border.TitledBorder +import kotlin.math.max +import kotlin.math.roundToInt + +/** + * GUI for editing the display range of volumes. + * Part of TransferFunctionEditor + * + * @author Jan Tiemann + */ +class DisplayRangeEditor(private val tfContainer: HasTransferFunction): JPanel(){ + + //RangeEditor + private val minText: JTextField + private val maxText: JTextField + private val rangeSlider: RangeSlider + private val minValueLabel: JLabel + private val maxValueLabel: JLabel + + init { + layout = MigLayout( + "insets 0, fill", + "[left, 10%]5[right, 40%]5[left, 10%]5[right, 40%]" + ) + val title = object: TitledBorder(MatteBorder(1, 0, 0, 0, Color.GRAY), "Display Range") { + val customInsets = Insets(25, 0, 25, 0) + override fun getBorderInsets(c: Component?): Insets { + return customInsets + } + + override fun getBorderInsets(c: Component?, insets: Insets?): Insets { + return customInsets + } + } + border = title + + // Range editor + val initMinValue = max(tfContainer.minDisplayRange.toInt(), 100) + minText = JTextField(initMinValue.toString()) + minValueLabel = JLabel(String.format("%.1f", tfContainer.range.first)) + + val initMaxValue = max(tfContainer.maxDisplayRange.toInt(), 100) + maxText = JTextField(initMaxValue.toString()) + maxText.horizontalAlignment = SwingConstants.RIGHT + maxValueLabel = JLabel(String.format("%.1f", tfContainer.range.second)) + + rangeSlider = RangeSlider() + rangeSlider.minimum = tfContainer.range.first.roundToInt() + rangeSlider.maximum = tfContainer.range.second.roundToInt() + rangeSlider.value = tfContainer.minDisplayRange.toInt() + rangeSlider.upperValue = tfContainer.maxDisplayRange.toInt() + + minText.addActionListener { updateSliderRange() } + maxText.addActionListener { updateSliderRange() } + rangeSlider.addChangeListener { + updateConverter() + } + + val rangeEditorPanel = this + rangeEditorPanel.add(JLabel("Minimum:"), "shrinkx") + rangeEditorPanel.add(minText, "growx") + rangeEditorPanel.add(JLabel("Maximum:"), "shrinkx") + rangeEditorPanel.add(maxText, "growx, wrap") + rangeEditorPanel.add(rangeSlider, "spanx, growx, wrap") + rangeEditorPanel.add(minValueLabel, "spanx 2, left") + rangeEditorPanel.add(maxValueLabel, "spanx 2, right") + } + + private fun updateSliderRange() { + val min = minText.toInt() + val max = maxText.toInt() + if (min != null && max != null) { + rangeSlider.value = min + rangeSlider.upperValue = max + } + updateConverter() + } + + private fun JTextField.toInt() = text.toIntOrNull() + + private fun updateConverter() { + minText.text = rangeSlider.value.toString() + maxText.text = rangeSlider.upperValue.toString() + minValueLabel.text = String.format("%.1f", tfContainer.range.first) + maxValueLabel.text = String.format("%.1f", tfContainer.range.second) + + tfContainer.minDisplayRange = rangeSlider.value.toFloat() + tfContainer.maxDisplayRange = rangeSlider.upperValue.toFloat() + } + + /** + * Returns the currently set display range. + */ + fun getDisplayRange(): Pair { + return rangeSlider.value.toFloat() to rangeSlider.upperValue.toFloat() + } + + /** + * Returns the current data range. + */ + fun getDataRange(): Pair { + return rangeSlider.minimum.toFloat() to rangeSlider.maximum.toFloat() + } + + /** + * Set tfcontainer values to gui. + */ + internal fun refreshDisplayRange() { + rangeSlider.value = tfContainer.minDisplayRange.toInt() + rangeSlider.upperValue = tfContainer.maxDisplayRange.toInt() + updateConverter() + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/DummyVolume.kt b/src/main/kotlin/graphics/scenery/volumes/DummyVolume.kt new file mode 100644 index 0000000000..927b647103 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/DummyVolume.kt @@ -0,0 +1,92 @@ +package graphics.scenery.volumes + +import bdv.tools.brightness.ConverterSetup +import graphics.scenery.* +import graphics.scenery.net.Networkable +import java.lang.IllegalArgumentException + +/** + * A container for the primary user parameters used in volume rendering. Can be used in server-side + * remote volume rendering applications to synchronize volume rendering parameters without transferring + * volume data between server and client. + */ +class DummyVolume(val counterStart : Int = 0) : DefaultNode("DummyVolume"), HasTransferFunction { + + /** The transfer function to use for the volume. Flat by default. */ + override var transferFunction: TransferFunction = TransferFunction.flat(0.5f) + set(m) { + field = m + modifiedAt = System.nanoTime() + } + var counter = 0 + + val converterSetups = ArrayList() + + /** Minimum display range. */ + override var minDisplayRange: Float = 0.0f + get() = field + set(value) { + setTransferFunctionRange(value, maxDisplayRange) + field = value + modifiedAt = System.nanoTime() + } + + /** Maximum display range. */ + override var maxDisplayRange: Float = 65535f + get() = field + set(value) { + setTransferFunctionRange(minDisplayRange, value) + field = value + modifiedAt = System.nanoTime() + } + + /** A pair containing the min and max display range. */ + override var range: Pair = Pair(minDisplayRange,maxDisplayRange) + get() = field + set(m) { + field = m + modifiedAt = System.nanoTime() + } + + /** The color map for the volume. */ + var colormap: Colormap = Colormap.get("viridis") + set(m) { + field = m + modifiedAt = System.nanoTime() + } + + init { + name = "DummyVolume" + counter = counterStart + } + + /** + * Update the DummyVolume with the [fresh] one received over the network. + */ + override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) { + if (fresh !is DummyVolume) throw IllegalArgumentException("Update called with object of foreign class") + super.update(fresh, getNetworkable, additionalData) + this.transferFunction = fresh.transferFunction + this.minDisplayRange = fresh.minDisplayRange + this.maxDisplayRange = fresh.maxDisplayRange + this.colormap = fresh.colormap + this.counter = fresh.counter + } + + /** + * Resets the range of this volume's transfer function to [min] and [max] for the setup given as [forSetupId]. + */ + @JvmOverloads + open fun setTransferFunctionRange(min: Float, max: Float, forSetupId: Int = 0) { + converterSetups.getOrNull(forSetupId)?.setDisplayRange(min.toDouble(), max.toDouble()) + } + + override fun getConstructorParameters(): Any? { + return counterStart + } + + override fun constructWithParameters(parameters: Any, hub: Hub): Networkable { + val counterStart = parameters as Int + return DummyVolume(counterStart) + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/HasHistogram.kt b/src/main/kotlin/graphics/scenery/volumes/HasHistogram.kt index 802e44a682..82bf523b00 100644 --- a/src/main/kotlin/graphics/scenery/volumes/HasHistogram.kt +++ b/src/main/kotlin/graphics/scenery/volumes/HasHistogram.kt @@ -1,16 +1,17 @@ package graphics.scenery.volumes -import net.imglib2.histogram.Histogram1d +import org.jfree.data.statistics.SimpleHistogramDataset /** * @author Konrad Michel + * @author Jan Tiemann * * Interface to abstract out a possible histogram implementation and untie it from pure Volume usage */ interface HasHistogram { /** - * This is a placeholder function that needs to be overwritten by the implementing class. Currently the output should be of type Histogram1d<*> - * from imglib2 + * @return most common value found in the histogram (aka. highest bar) or null if no histogram could be generated. + * Needed for scaling the display correctly. */ - fun generateHistogram() : Histogram1d<*>? + fun generateHistogram(volumeHistogramData: SimpleHistogramDataset): Int? } diff --git a/src/main/kotlin/graphics/scenery/volumes/HasTransferFunction.kt b/src/main/kotlin/graphics/scenery/volumes/HasTransferFunction.kt index 07b9b77a8b..ff51788ccb 100644 --- a/src/main/kotlin/graphics/scenery/volumes/HasTransferFunction.kt +++ b/src/main/kotlin/graphics/scenery/volumes/HasTransferFunction.kt @@ -1,7 +1,13 @@ package graphics.scenery.volumes +import java.io.BufferedWriter +import java.io.File +import java.io.FileWriter +import java.io.InputStream + /** * @author Konrad Michel + * @author Jan Tiemann * * Interface to abstract out the basic parameters of a TransferFunction use case. */ @@ -11,4 +17,43 @@ interface HasTransferFunction { var minDisplayRange : Float var maxDisplayRange : Float var range: Pair + + /** + * Load transfer function and display range from file that was written by [HasTransferFunction.saveTransferFunctionToFile] + */ + fun loadTransferFunctionFromFile(file: File){ + val tf = TransferFunction() + val inputStream: InputStream = file.inputStream() + var isRangeSet = false + inputStream.bufferedReader().forEachLine { + val line = it.trim().split(";").mapNotNull(kotlin.String::toFloatOrNull) + if (line.size == 2){ + if (!isRangeSet){ + minDisplayRange = line[0] + maxDisplayRange = line[1] + isRangeSet = true + } else { + tf.addControlPoint(line[0], line[1]) + } + } + } + transferFunction = tf + } + + /** + * Write transfer function to file in a human-readable way. + * Format: + * First line is the display range sepaerated by a semicolon. + * All following lines are tf control points. + */ + fun saveTransferFunctionToFile(file: File){ + val writer = BufferedWriter(FileWriter(file)) + writer.write("${minDisplayRange};${maxDisplayRange}") + writer.newLine() + transferFunction.controlPoints().forEach { + writer.write("${it.value};${it.factor}") + writer.newLine() + } + writer.close() + } } diff --git a/src/main/kotlin/graphics/scenery/volumes/HistogramChartManager.kt b/src/main/kotlin/graphics/scenery/volumes/HistogramChartManager.kt new file mode 100644 index 0000000000..e4591f9359 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/HistogramChartManager.kt @@ -0,0 +1,91 @@ +package graphics.scenery.volumes + +import org.jfree.chart.ChartPanel +import org.jfree.chart.axis.LogarithmicAxis +import org.jfree.chart.axis.NumberAxis +import org.jfree.chart.plot.XYPlot +import org.jfree.chart.renderer.xy.StandardXYBarPainter +import org.jfree.chart.renderer.xy.XYBarRenderer +import org.jfree.data.statistics.SimpleHistogramDataset +import java.awt.Color +import javax.swing.JCheckBox +import kotlin.math.abs + +/** + * Handles all histogram chart related things. + * + * @author Jan Tiemann + */ +class HistogramChartManager(val tfPlot: XYPlot, + val mainChart: ChartPanel, + private val tfContainer: HasTransferFunction, + val axisExtensionFactor: Double){ + val genHistButton = JCheckBox("Show Histogram") + + private val histYAxis = LogarithmicAxis("") + private val histXAxis = NumberAxis() + + init { + val histogramRenderer = XYBarRenderer() + histogramRenderer.setShadowVisible(false) + histogramRenderer.barPainter = StandardXYBarPainter() + histogramRenderer.isDrawBarOutline = false + histogramRenderer.setSeriesPaint(0, Color(160,160,255)) + tfPlot.setRenderer(1, histogramRenderer) + + + histXAxis.isTickLabelsVisible = false + histXAxis.isMinorTickMarksVisible = false + histXAxis.isTickMarksVisible = false + histXAxis.autoRangeIncludesZero = false + histXAxis.autoRangeStickyZero = false + histXAxis.isAutoRange = false + + histYAxis.isTickLabelsVisible = false + histYAxis.isMinorTickMarksVisible = false + histYAxis.isTickMarksVisible = false + + val volumeHistogramData = SimpleHistogramDataset("VolumeBin") + volumeHistogramData.adjustForBinSize = false + + if (tfContainer is HasHistogram) { + genHistButton.addActionListener { + val histogramVisible = tfPlot.getDataset(1) != null + + if(histogramVisible) { + // hide histogram + tfPlot.setDataset(1, null) + tfPlot.setDomainAxis(1, null) + tfPlot.setRangeAxis(1, null) + + mainChart.repaint() + } else { + tfPlot.setDataset(1, volumeHistogramData) + tfPlot.setRangeAxis(1, histYAxis) + generateHistogram( volumeHistogramData) + + tfPlot.setDomainAxis(1, histXAxis) + + mainChart.repaint() + } + } + } + } + + private fun generateHistogram(volumeHistogramData: SimpleHistogramDataset) { + volumeHistogramData.removeAllBins() + val max = (tfContainer as? HasHistogram)?.generateHistogram(volumeHistogramData) ?: return + + histYAxis.setRange( + -0.1 , + max * (1.2) + ) + + val displayRange = abs(tfContainer.maxDisplayRange - tfContainer.minDisplayRange) + histXAxis.setRange( + tfContainer.minDisplayRange - (axisExtensionFactor * displayRange), + tfContainer.maxDisplayRange + (axisExtensionFactor * displayRange) + ) + } + +} diff --git a/src/main/kotlin/graphics/scenery/volumes/RaycastingPropertiesEditor.kt b/src/main/kotlin/graphics/scenery/volumes/RaycastingPropertiesEditor.kt new file mode 100644 index 0000000000..2b1fa5d4bb --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/RaycastingPropertiesEditor.kt @@ -0,0 +1,67 @@ +package graphics.scenery.volumes + +import net.miginfocom.swing.MigLayout +import javax.swing.* + + +/** + * Provides a graphical editor to enable interactively changing volume raycasting properties (e.g., the step size + * along the ray) in volume rendering applications. + * + * @author Aryaman Gupta + */ +class RaycastingPropertiesEditor constructor( + private val volumeManager: VolumeManager +): JPanel() { + + init { + layout = MigLayout("flowy") + + //Toggling fixedStepSize + val fixedSizeButton = JCheckBox("Fixed step size") + add(fixedSizeButton, "growx") + + val stepsText = JTextField("2.00") + stepsText.isEnabled = false + + fixedSizeButton.addActionListener { + volumeManager.shaderProperties["fixedStepSize"] = fixedSizeButton.isSelected + stepsText.isEnabled = fixedSizeButton.isSelected + } + + val editorPanel = JPanel() + editorPanel.layout = MigLayout("fill") + add(editorPanel, "grow") + + editorPanel.add(fixedSizeButton) + editorPanel.add(JLabel("Steps per voxel:"), "shrinkx") + editorPanel.add(stepsText) + + stepsText.addActionListener { + try { + volumeManager.shaderProperties["stepsPerVoxel"] = stepsText.text.toFloat() + } catch (_: NumberFormatException) { + + } + } + + } + + /** + * Companion object as described in https://kotlinlang.org/docs/object-declarations.html#companion-objects + */ + companion object { + /** + * Convenience function to open a JFrame containing a [RaycastingPropertiesEditor] + */ + fun show(volumeManager: VolumeManager): JFrame { + val frame = JFrame() + frame.title = "Raycasting properties" + val editor = RaycastingPropertiesEditor(volumeManager) + frame.add(editor) + frame.pack() + frame.isVisible = true + return frame + } + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/RemoteRenderingProperties.kt b/src/main/kotlin/graphics/scenery/volumes/RemoteRenderingProperties.kt new file mode 100644 index 0000000000..bccc305615 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/RemoteRenderingProperties.kt @@ -0,0 +1,40 @@ +package graphics.scenery.volumes + +import graphics.scenery.DefaultNode +import graphics.scenery.net.Networkable + +/** + * A node used to synchronize remote rendering properties between server and client for remote volume + * rendering. + */ +class RemoteRenderingProperties : DefaultNode("RemoteRenderingProperties"), Networkable { + + + /** + * The types of streaming supported for remote volume rendering. + */ + enum class StreamType { + VolumeRendering, + VDI, + None + } + + /** + * The streaming type used by this object currently. + */ + var streamType : StreamType = StreamType.VolumeRendering + set(value){ + field = value + modifiedAt = System.nanoTime() + } + + init { + name = "RemoteRenderingProperties" + } + + override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) { + super.update(fresh, getNetworkable,additionalData) + if (fresh !is RemoteRenderingProperties) throw IllegalArgumentException("Update called with object of foreign class") + this.streamType = fresh.streamType + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt b/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt index cd36fead9a..984a2355c5 100644 --- a/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt +++ b/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt @@ -343,7 +343,8 @@ open class SceneryContext(val node: VolumeManager, val useCompute: Boolean = fal } val repeat = when(texture.texWrap()) { - BVVTexture.Wrap.CLAMP_TO_BORDER_ZERO -> RepeatMode.ClampToBorder + // TODO: Fix this in BigVolumeViewer + BVVTexture.Wrap.CLAMP_TO_BORDER_ZERO -> RepeatMode.ClampToEdge BVVTexture.Wrap.CLAMP_TO_EDGE -> RepeatMode.ClampToEdge BVVTexture.Wrap.REPEAT -> RepeatMode.Repeat else -> throw UnsupportedOperationException("Unknown wrapping mode: ${texture.texWrap()}") diff --git a/src/main/kotlin/graphics/scenery/volumes/TransferFunction.kt b/src/main/kotlin/graphics/scenery/volumes/TransferFunction.kt index 2f91e72016..6e95dfe8d4 100644 --- a/src/main/kotlin/graphics/scenery/volumes/TransferFunction.kt +++ b/src/main/kotlin/graphics/scenery/volumes/TransferFunction.kt @@ -2,6 +2,10 @@ package graphics.scenery.volumes import graphics.scenery.utils.lazyLogger import org.lwjgl.system.MemoryUtil +import java.io.BufferedWriter +import java.io.File +import java.io.FileWriter +import java.io.InputStream import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.concurrent.CopyOnWriteArrayList @@ -9,6 +13,7 @@ import kotlin.math.abs import kotlin.math.max import kotlin.math.min + /** Transfer function class with an optional [name]. */ open class TransferFunction(val name: String = "") { private val logger by lazyLogger() @@ -34,6 +39,11 @@ open class TransferFunction(val name: String = "") { @Transient var stale = true protected set + /** + * @return control points of this tf + */ + fun controlPoints() = controlPoints.toList() + /** * Adds a new control point for position [value], with [factor]. */ diff --git a/src/main/kotlin/graphics/scenery/volumes/TransferFunctionEditor.kt b/src/main/kotlin/graphics/scenery/volumes/TransferFunctionEditor.kt index 4bfd7b282a..8d1e13cf03 100644 --- a/src/main/kotlin/graphics/scenery/volumes/TransferFunctionEditor.kt +++ b/src/main/kotlin/graphics/scenery/volumes/TransferFunctionEditor.kt @@ -1,23 +1,16 @@ package graphics.scenery.volumes -import graphics.scenery.ui.RangeSlider import net.miginfocom.swing.MigLayout import org.jfree.chart.ChartMouseEvent import org.jfree.chart.ChartMouseListener import org.jfree.chart.ChartPanel import org.jfree.chart.JFreeChart import org.jfree.chart.annotations.XYTextAnnotation -import org.jfree.chart.axis.LogarithmicAxis -import org.jfree.chart.axis.NumberAxis -import org.jfree.chart.axis.NumberTickUnit +import org.jfree.chart.axis.* import org.jfree.chart.entity.XYItemEntity import org.jfree.chart.labels.XYToolTipGenerator import org.jfree.chart.plot.XYPlot -import org.jfree.chart.renderer.xy.StandardXYBarPainter -import org.jfree.chart.renderer.xy.XYBarRenderer import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer -import org.jfree.data.statistics.SimpleHistogramBin -import org.jfree.data.statistics.SimpleHistogramDataset import org.jfree.data.xy.XYDataset import org.jfree.data.xy.XYSeries import org.jfree.data.xy.XYSeriesCollection @@ -25,16 +18,18 @@ import org.joml.Math.clamp import java.awt.Color import java.awt.Cursor import java.awt.Dimension +import java.awt.Image.SCALE_SMOOTH import java.awt.event.MouseEvent +import java.awt.event.MouseEvent.BUTTON1 +import java.awt.event.MouseEvent.BUTTON1_DOWN_MASK import java.awt.event.MouseListener import java.awt.event.MouseMotionListener import java.awt.image.BufferedImage import java.text.NumberFormat +import javax.imageio.ImageIO import javax.swing.* -import kotlin.math.abs -import kotlin.math.max -import kotlin.math.pow -import kotlin.math.roundToInt +import javax.swing.event.PopupMenuEvent +import javax.swing.event.PopupMenuListener /** @@ -45,9 +40,8 @@ import kotlin.math.roundToInt * Able to generate a histogram and visualize it as well to help with TF-settings * Able to dynamically set the transfer function range -> changes histogram as well */ -class TransferFunctionEditor constructor( - private val tfContainer: HasTransferFunction, - volumeName: String = "Volume" +class TransferFunctionEditor( + private val tfContainer: HasTransferFunction ): JPanel() { /** * MouseDragTarget is set when a ControlPoint has been clicked. The initial index is set to -1 and reset when the Controlpoint has been deleted @@ -56,25 +50,16 @@ class TransferFunctionEditor constructor( data class MouseDragTarget( var seriesIndex: Int = -1, var itemIndex: Int = -1, - var lastIndex: Int = -1, var x: Double = 0.0, var y: Double = 0.0 ) private val mouseTargetCP = MouseDragTarget() + private val displayRangeEditor = DisplayRangeEditor(tfContainer) //TFEditor and Histogram val mainChart: JPanel - //RangeEditor - private val rangeEditorPanel: JPanel - private val minText: JTextField - private val maxText: JTextField - private val rangeSlider: RangeSlider - private val minValueLabel: JLabel - private val maxValueLabel: JLabel - - private class ValueAlphaTooltipGenerator : XYToolTipGenerator { override fun generateToolTip(dataset: XYDataset, series: Int, category: Int): String { val x: Number = dataset.getXValue(series, category) @@ -85,7 +70,7 @@ class TransferFunctionEditor constructor( init { - layout = MigLayout("flowy") + layout = MigLayout("", "[][][][]") // MainChart manipulation val tfCollection = XYSeriesCollection() @@ -100,29 +85,7 @@ class TransferFunctionEditor constructor( val tfRenderer = XYLineAndShapeRenderer() tfPlot.setRenderer(0, tfRenderer) - val histogramRenderer = XYBarRenderer() - histogramRenderer.setShadowVisible(false) - histogramRenderer.barPainter = StandardXYBarPainter() - histogramRenderer.isDrawBarOutline = false - tfPlot.setRenderer(1, histogramRenderer) - - val histXAxis = NumberAxis() - var range = abs(tfContainer.maxDisplayRange - tfContainer.minDisplayRange) val axisExtensionFactor = 0.02 - histXAxis.setRange( - tfContainer.minDisplayRange - (axisExtensionFactor * range), - tfContainer.maxDisplayRange + (axisExtensionFactor * range) - ) - - val histogramAxis = LogarithmicAxis("") - histogramAxis.isMinorTickMarksVisible = true - val histHeight = abs(1000.0 - 0.0) - histogramAxis.setRange( - 0.0 - (axisExtensionFactor / 100.0 * histHeight), - 1000.0 + (axisExtensionFactor * histHeight) - ) - histogramAxis.allowNegativesFlag - val tfYAxis = NumberAxis() tfYAxis.setRange(0.0 - axisExtensionFactor, 1.0 + axisExtensionFactor) tfYAxis.tickUnit = NumberTickUnit(0.1) @@ -130,6 +93,16 @@ class TransferFunctionEditor constructor( val tfXAxis = NumberAxis() tfXAxis.setRange(0.0 - axisExtensionFactor, 1.0 + axisExtensionFactor) + val units = TickUnits() + units.add(object: NumberTickUnit(0.1, NumberFormat.getIntegerInstance()) { + override fun valueToString(value: Double): String { + val r = displayRangeEditor.getDisplayRange() + return super.valueToString(value * (r.second - r.first) + r.first) + } + }) + tfXAxis.standardTickUnits = units + + tfPlot.setDomainAxis(0, tfXAxis) tfPlot.mapDatasetToRangeAxis(0, 0) tfPlot.mapDatasetToDomainAxis(0, 0) @@ -159,22 +132,26 @@ class TransferFunctionEditor constructor( mainChart.cursor = Cursor(Cursor.CROSSHAIR_CURSOR) - add(mainChart, "grow") + add(mainChart, "grow,wrap") mainChart.removeMouseMotionListener(mainChart) mainChart.addMouseListener(object : MouseListener { override fun mouseReleased(e: MouseEvent) { mouseTargetCP.itemIndex = -1 } - override fun mousePressed(e: MouseEvent) {} - override fun mouseClicked(e: MouseEvent) {} - override fun mouseEntered(e: MouseEvent) {} - override fun mouseExited(e: MouseEvent) {} + override fun mousePressed(e: MouseEvent) {/*noop*/} + override fun mouseClicked(e: MouseEvent) {/*noop*/} + override fun mouseEntered(e: MouseEvent) {/*noop*/} + override fun mouseExited(e: MouseEvent) {/*noop*/} }) var lastUpdate = 0L mainChart.addMouseMotionListener(object : MouseMotionListener { override fun mouseDragged(e: MouseEvent) { + if(!SwingUtilities.isLeftMouseButton(e)) { + return + } + val chart = e.component as ChartPanel val point = mainChart.translateJava2DToScreen(e.point) val item = chart.getEntityForPoint(point.x, point.y) @@ -184,7 +161,6 @@ class TransferFunctionEditor constructor( if (item.dataset is XYSeriesCollection) { mouseTargetCP.seriesIndex = item.seriesIndex mouseTargetCP.itemIndex = item.item - mouseTargetCP.lastIndex = item.item } } //if the drag is performed while the current target is indeed set to be a CP, update it @@ -198,7 +174,7 @@ class TransferFunctionEditor constructor( val annotation = XYTextAnnotation( "%.2f / %.2f".format(mouseTargetCP.x.toFloat(), mouseTargetCP.y.toFloat()), mouseTargetCP.x, - mouseTargetCP.y) + mouseTargetCP.y-0.04) //offset so the label does not hide the target annotation.backgroundPaint = Color.white annotation.paint = Color.darkGray tfPlot.clearAnnotations() @@ -211,148 +187,125 @@ class TransferFunctionEditor constructor( } } } - override fun mouseMoved(e: MouseEvent) {} + override fun mouseMoved(e: MouseEvent) {/*noop*/} }) - - mainChart.addChartMouseListener(object : ChartMouseListener { override fun chartMouseClicked(e: ChartMouseEvent) { + if(!SwingUtilities.isLeftMouseButton(e.trigger)) { + return + } + if (e.entity is XYItemEntity) { val item = e.entity as XYItemEntity //click on cp if (item.dataset is XYSeriesCollection) { mouseTargetCP.seriesIndex = item.seriesIndex mouseTargetCP.itemIndex = item.item - mouseTargetCP.lastIndex = item.item mouseTargetCP.x = clamp(0.0, 1.0, item.dataset.getX(item.seriesIndex, item.item).toDouble()) mouseTargetCP.y = clamp(0.0, 1.0, item.dataset.getY(item.seriesIndex, item.item).toDouble()) - if (e.trigger.isControlDown && mouseTargetCP.itemIndex != -1) { + if ((e.trigger.clickCount > 1 || e.trigger.isControlDown) && mouseTargetCP.itemIndex != -1) { removeControlpoint(mouseTargetCP) tfPlot.backgroundImage = createTFImage() } - } - //click on histogram - else { - val point = mainChart.translateJava2DToScreen(e.trigger.point) - val plotArea = mainChart.chartRenderingInfo.plotInfo.dataArea - mouseTargetCP.x = clamp( - 0.0, - 1.0, - tfPlot.getDomainAxis(0).java2DToValue(point.getX(), plotArea, tfPlot.domainAxisEdge) - ) - mouseTargetCP.y = clamp( - 0.0, - 1.0, - tfPlot.getRangeAxis(0).java2DToValue(point.getY(), plotArea, tfPlot.rangeAxisEdge) - ) - - if (mouseTargetCP.itemIndex == -1) { - addControlpoint(mouseTargetCP) - tfPlot.backgroundImage = createTFImage() - } + return } } - //click on empty region - else { - val point = mainChart.translateJava2DToScreen(e.trigger.point) - val plotArea = mainChart.chartRenderingInfo.plotInfo.dataArea - mouseTargetCP.x = clamp( - 0.0, - 1.0, - tfPlot.getDomainAxis(0).java2DToValue(point.getX(), plotArea, tfPlot.domainAxisEdge) - ) - mouseTargetCP.y = clamp( - 0.0, - 1.0, - tfPlot.getRangeAxis(0).java2DToValue(point.getY(), plotArea, tfPlot.rangeAxisEdge) - ) - - if (mouseTargetCP.itemIndex == -1) { - addControlpoint(mouseTargetCP) - tfPlot.backgroundImage = createTFImage() - } + + //click on graph or empty region + val point = mainChart.translateJava2DToScreen(e.trigger.point) + val plotArea = mainChart.chartRenderingInfo.plotInfo.dataArea + mouseTargetCP.x = clamp( + 0.0, + 1.0, + tfPlot.getDomainAxis(0).java2DToValue(point.getX(), plotArea, tfPlot.domainAxisEdge) + ) + mouseTargetCP.y = clamp( + 0.0, + 1.0, + tfPlot.getRangeAxis(0).java2DToValue(point.getY(), plotArea, tfPlot.rangeAxisEdge) + ) + + if (mouseTargetCP.itemIndex == -1) { + addControlpoint(mouseTargetCP) + tfPlot.backgroundImage = createTFImage() } + } - override fun chartMouseMoved(e: ChartMouseEvent) {} + override fun chartMouseMoved(e: ChartMouseEvent) {/*noop*/} }) - //Histogram Manipulation - val genHistButton = JCheckBox("Show Histogram") - add(genHistButton, "growx") - - val volumeHistogramData = SimpleHistogramDataset("VolumeBin") - volumeHistogramData.adjustForBinSize = false - val resolutionStartExp = 8 - val binResolution = 2.0.pow(resolutionStartExp) - - if (tfContainer is HasHistogram) { - genHistButton.addChangeListener { - val histogramVisible = tfPlot.getDataset(1) != null + val histAndTFIOButtonsPanel = JPanel() + histAndTFIOButtonsPanel.layout = MigLayout("insets 0", "[][][][]") + add(histAndTFIOButtonsPanel, "growx,wrap") + + val histogramChartManager = HistogramChartManager(tfPlot,mainChart,tfContainer,axisExtensionFactor) + histAndTFIOButtonsPanel.add(histogramChartManager.genHistButton, "growx") + + // transfer function IO + val fc = JFileChooser() + val tfMenu = JPopupMenu() + tfMenu.add(JMenuItem("Load transfer function ...").also { + it.addActionListener { + val returnVal: Int = fc.showOpenDialog(this) + if(returnVal == JFileChooser.APPROVE_OPTION) { + tfContainer.loadTransferFunctionFromFile(file = fc.selectedFile) + initTransferFunction(tfContainer.transferFunction) + displayRangeEditor.refreshDisplayRange() + } + } - if(histogramVisible) { - tfPlot.setDataset(1, null) - tfPlot.setDomainAxis(1, null) - tfPlot.setRangeAxis(1, null) + }) + tfMenu.add(JMenuItem("Save transfer function ...").also { + it.toolTipText = "Save the current transfer function and display range to a file" + it.addActionListener { + val option = fc.showSaveDialog(this) + if (option == JFileChooser.APPROVE_OPTION) { + tfContainer.saveTransferFunctionToFile(fc.selectedFile) + } + } + }) + tfMenu.add(JMenuItem("Reset transfer function").also { + it.addActionListener { + tfContainer.transferFunction = TransferFunction.flat(0.5f) + initTransferFunction(tfContainer.transferFunction) + tfPlot.backgroundImage = createTFImage() + } + }) - mainChart.repaint() + val tfMenuButton = JToggleButton("").also { button -> + button.icon = ImageIcon(ImageIcon(ImageIO.read(this::class.java.getResource("/graphics/scenery/ui/gear.png"))).image.getScaledInstance(16, 16, SCALE_SMOOTH)) + button.toolTipText = "Load a new transfer function and display range" + button.addActionListener { + if(button.isSelected) { + tfMenu.show(button, 0, button.height) } else { - tfPlot.setDataset(1, volumeHistogramData) - tfPlot.setRangeAxis(1, histogramAxis) - generateHistogramBins(binResolution, volumeHistogramData) - range = abs(tfContainer.maxDisplayRange - tfContainer.minDisplayRange) - histXAxis.setRange( - tfContainer.minDisplayRange - (axisExtensionFactor * range), - tfContainer.maxDisplayRange + (axisExtensionFactor * range) - ) - - histogramAxis.setRange( - 0.0 - (axisExtensionFactor / 100.0 * histHeight), - 1000.0 + (axisExtensionFactor * histHeight) - ) - tfPlot.setDomainAxis(1, histXAxis) - - mainChart.repaint() + tfMenu.isVisible = false } } + histAndTFIOButtonsPanel.add(button, "skip 2, al right, push") } - // Range editor - val initMinValue = max(tfContainer.minDisplayRange.toInt(), 100) - minText = JTextField(initMinValue.toString()) - minValueLabel = JLabel(String.format("%.1f", tfContainer.range.first)) - - val initMaxValue = max(tfContainer.maxDisplayRange.toInt(), 100) - maxText = JTextField(initMaxValue.toString()) - maxText.horizontalAlignment = SwingConstants.RIGHT - maxValueLabel = JLabel(String.format("%.1f", tfContainer.range.second)) - - rangeSlider = RangeSlider() - rangeSlider.minimum = tfContainer.range.first.roundToInt() - rangeSlider.maximum = tfContainer.range.second.roundToInt() - rangeSlider.value = tfContainer.minDisplayRange.toInt() - rangeSlider.upperValue = tfContainer.maxDisplayRange.toInt() - - minText.addActionListener { updateSliderRange() } - maxText.addActionListener { updateSliderRange() } - rangeSlider.addChangeListener { - updateConverter() - } + tfMenu.addPopupMenuListener(object: PopupMenuListener { + override fun popupMenuWillBecomeVisible(e: PopupMenuEvent?) { + /* not used */ + } - rangeEditorPanel = JPanel() - rangeEditorPanel.layout = MigLayout("fill", - "[left, 10%]5[right, 40%]5[left, 10%]5[right, 40%]") - add(rangeEditorPanel, "grow") + override fun popupMenuWillBecomeInvisible(e: PopupMenuEvent?) { + tfMenuButton.isSelected = false + } + + override fun popupMenuCanceled(e: PopupMenuEvent?) { + tfMenuButton.isSelected = false + } + + }) - rangeEditorPanel.add(JLabel("min:"), "shrinkx") - rangeEditorPanel.add(minText, "growx") - rangeEditorPanel.add(JLabel("max:"), "shrinkx") - rangeEditorPanel.add(maxText, "growx, wrap") - rangeEditorPanel.add(rangeSlider, "spanx, growx, wrap") - rangeEditorPanel.add(minValueLabel, "spanx 2, left") - rangeEditorPanel.add(maxValueLabel, "spanx 2, right") + initTransferFunction(tfContainer.transferFunction) -// updateSliderRange() + + add(displayRangeEditor, "grow,wrap") + add(ColormapPanel(tfContainer as? Volume), "grow,wrap") } private fun createTFImage(): BufferedImage { @@ -377,28 +330,6 @@ class TransferFunctionEditor constructor( return tfImage } - private fun updateSliderRange() { - val min = minText.toInt() - val max = maxText.toInt() - if (min != null && max != null) { - rangeSlider.value = min - rangeSlider.upperValue = max - } - updateConverter() - } - - private fun updateConverter() { - minText.text = rangeSlider.value.toString() - maxText.text = rangeSlider.upperValue.toString() - minValueLabel.text = String.format("%.1f", tfContainer.range.first) - maxValueLabel.text = String.format("%.1f", tfContainer.range.second) - - tfContainer.minDisplayRange = rangeSlider.value.toFloat() - tfContainer.maxDisplayRange = rangeSlider.upperValue.toFloat() - } - - private fun JTextField.toInt() = text.toIntOrNull() - private fun addControlpoint(targetCP: MouseDragTarget) { val chart = mainChart as ChartPanel val collection = chart.chart.xyPlot.getDataset(0) as XYSeriesCollection @@ -407,11 +338,28 @@ class TransferFunctionEditor constructor( series.add( targetCP.x.toFloat(), targetCP.y.toFloat() ) - val newTF = TransferFunction() - for (i in 0 until series.itemCount) { - newTF.addControlPoint(series.getX(i).toFloat(), series.getY(i).toFloat()) + regenerateTF(series) + } + + private fun initTransferFunction(transferFunction: TransferFunction){ + val chart = mainChart as ChartPanel + val collection = chart.chart.xyPlot.getDataset(0) as XYSeriesCollection + val series = collection.getSeries("ControlPoints") + series.clear() + + var points = transferFunction.controlPoints().map { it.value to it.factor } + + // add first and last point if not there + if ((points.firstOrNull()?.first ?: 1f) > 0.0f){ + points = listOf(0f to 0f) + points + } + if ((points.lastOrNull()?.first ?: 0f) < 1.0f){ + points = listOf(1f to 1f) + points + } + + points.forEach { + series.add(it.first,it.second) } - tfContainer.transferFunction = newTF } private fun updateControlpoint(targetCP: MouseDragTarget) { @@ -419,17 +367,28 @@ class TransferFunctionEditor constructor( val collection = chart.chart.xyPlot.getDataset(0) as XYSeriesCollection val series = collection.getSeries(targetCP.seriesIndex) - targetCP.x = clamp(0.0, 1.0, targetCP.x) + // dont move point past other points + val epsilon = 0.005 + val minX = if (targetCP.itemIndex > 0) { + val prev = series.getDataItem(targetCP.itemIndex - 1) + prev.x.toDouble() + epsilon + } else { + 0.0 + } + val maxX = if (targetCP.itemIndex < series.itemCount-1) { + val prev = series.getDataItem(targetCP.itemIndex + 1) + prev.x.toDouble() - epsilon + } else { + 1.0 + } + + targetCP.x = clamp(minX, maxX, targetCP.x) targetCP.y = clamp(0.0, 1.0, targetCP.y) series.remove(targetCP.itemIndex) series.add(targetCP.x, targetCP.y) - val newTF = TransferFunction() - for (i in 0 until series.itemCount) { - newTF.addControlPoint(series.getX(i).toFloat(), series.getY(i).toFloat()) - } - tfContainer.transferFunction = newTF + regenerateTF(series) } private fun removeControlpoint(targetCP: MouseDragTarget) { @@ -438,48 +397,25 @@ class TransferFunctionEditor constructor( val series = collection.getSeries(targetCP.seriesIndex) - series.remove(targetCP.lastIndex) + series.remove(targetCP.itemIndex) + regenerateTF(series) + + targetCP.itemIndex = -1 + } + + private fun regenerateTF(series: XYSeries) { val newTF = TransferFunction() for (i in 0 until series.itemCount) { newTF.addControlPoint(series.getX(i).toFloat(), series.getY(i).toFloat()) } tfContainer.transferFunction = newTF - - targetCP.lastIndex = -1 - } - - private fun generateHistogramBins(binCount: Double, volumeHistogramData: SimpleHistogramDataset) { - volumeHistogramData.removeAllBins() - - val histogram = (tfContainer as? HasHistogram)?.generateHistogram() - if (histogram != null) { - var binEnd = -0.0000001 - val displayRange = abs(tfContainer.maxDisplayRange - tfContainer.minDisplayRange) - val binSize = displayRange / binCount - histogram.forEachIndexed { index, longType -> - - val relativeCount = (longType.get().toFloat() / histogram.totalCount().toFloat()) * histogram.binCount - val value = - (((index.toDouble() / histogram.binCount.toDouble()) * (displayRange / histogram.binCount.toDouble()))) * histogram.binCount.toDouble() - - if (relativeCount.roundToInt() != 0 && (value) >= binEnd) { - val binStart = - (((index) - (((index) % (histogram.binCount.toDouble() / binCount)))) / histogram.binCount.toDouble()) * displayRange - binEnd = binStart + binSize - val bin = SimpleHistogramBin(binStart, binEnd, true, false) - volumeHistogramData.addBin(bin) - } - for (i in 0 until relativeCount.roundToInt()) { - volumeHistogramData.addObservation(value) - } - } - } } companion object{ fun showTFFrame(tfContainer: HasTransferFunction, volumeName: String = "Volume"){ val frame = JFrame() - val tfe = TransferFunctionEditor(tfContainer,volumeName) + frame.title = "$volumeName transfer function" + val tfe = TransferFunctionEditor(tfContainer) frame.add(tfe) frame.pack() frame.isVisible = true diff --git a/src/main/kotlin/graphics/scenery/volumes/Volume.kt b/src/main/kotlin/graphics/scenery/volumes/Volume.kt index 92aab7e218..bb33c69d69 100644 --- a/src/main/kotlin/graphics/scenery/volumes/Volume.kt +++ b/src/main/kotlin/graphics/scenery/volumes/Volume.kt @@ -42,10 +42,10 @@ import io.scif.filters.ReaderFilter import io.scif.util.FormatTools import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription import mpicbg.spim.data.sequence.FinalVoxelDimensions +import net.imagej.ops.OpService import net.imglib2.RandomAccessibleInterval import net.imglib2.Volatile import net.imglib2.histogram.Histogram1d -import net.imglib2.histogram.Real1dBinMapper import net.imglib2.realtransform.AffineTransform3D import net.imglib2.type.numeric.ARGBType import net.imglib2.type.numeric.NumericType @@ -67,13 +67,18 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicInteger import kotlin.io.path.name -import kotlin.math.abs -import kotlin.math.max -import kotlin.math.min -import kotlin.math.sqrt import kotlin.properties.Delegates -import kotlin.streams.toList import net.imglib2.type.numeric.RealType +import net.imglib2.type.volatiles.VolatileByteType +import net.imglib2.type.volatiles.VolatileFloatType +import net.imglib2.type.volatiles.VolatileShortType +import net.imglib2.type.volatiles.VolatileUnsignedByteType +import net.imglib2.type.volatiles.VolatileUnsignedShortType +import org.jfree.data.statistics.SimpleHistogramBin +import org.jfree.data.statistics.SimpleHistogramDataset +import org.scijava.Context +import kotlin.math.* +import kotlin.time.measureTimedValue @Suppress("DEPRECATION") open class Volume( @@ -88,6 +93,9 @@ open class Volume( // without this line the *java* serialization framework kryo does not recognize the parameter-less constructor // and uses dark magic to instanciate this class + + var wantsSync = true + override fun wantsSync(): Boolean = wantsSync constructor() : this(VolumeDataSource.NullSource, hub = Hub("dummyVolumeHub")) var initalizer: VolumeInitializer? = null @@ -211,6 +219,15 @@ open class Volume( field = value } + val bytesPerVoxel: Int + get() { + return when(dataSource){ + VolumeDataSource.NullSource -> 1 + is VolumeDataSource.RAISource<*> -> dataSource.type.toBytesPerValue() + is SpimDataMinimalSource -> (dataSource.sources.first().spimSource.type as NumericType<*>).toBytesPerValue() + } + } + sealed class VolumeDataSource { class SpimDataMinimalSource( @Transient @@ -357,15 +374,37 @@ open class Volume( private fun NumericType<*>.toRange(): Pair { return when(this) { is UnsignedByteType -> 0.0f to 255.0f + is VolatileUnsignedByteType -> 0.0f to 255.0f is ByteType -> -127.0f to 128.0f + is VolatileByteType -> -127.0f to 128.0f is UnsignedShortType -> 0.0f to 65535.0f + is VolatileUnsignedShortType -> 0.0f to 65535.0f is ShortType -> -32768.0f to 32767.0f + is VolatileShortType -> -32768.0f to 32767.0f is FloatType -> 0.0f to 1.0f + is VolatileFloatType -> 0.0f to 1.0f else -> 0.0f to 1.0f } } + private fun NumericType<*>.toBytesPerValue(): Int { + return when(this) { + is UnsignedByteType -> 1 + is VolatileUnsignedByteType -> 1 + is ByteType -> 1 + is VolatileByteType -> 1 + is UnsignedShortType -> 2 + is VolatileUnsignedShortType -> 2 + is ShortType -> 2 + is VolatileShortType -> 2 + is FloatType -> 4 + is VolatileFloatType -> 4 + else -> 4 + } + } + + override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) { if (fresh !is Volume) throw IllegalArgumentException("Update called with object of foreign class") super.update(fresh, getNetworkable, additionalData) @@ -402,37 +441,104 @@ open class Volume( return VolumeSpatial(this) } + + /** + * Calculates the histogram on the CPU. + */ + override fun generateHistogram(volumeHistogramData: SimpleHistogramDataset): Int? { + volumeHistogramData.removeAllBins() + val bins = 1024 + // This generates a histogram over the whole volume ignoring the display range. + val absoluteHistogram = generateHistogramSPIMSourceOnCPU(512, bins) + if (absoluteHistogram != null) { + // We now need to select only the bins we care about. + val absoluteBinSize = absoluteHistogram.max() / bins.toDouble() + val minDisplayRange = minDisplayRange.toDouble() + val maxDisplayRange = maxDisplayRange.toDouble() + + var max = 100 + absoluteHistogram.forEachIndexed { index, longType -> + val startOfAbsoluteBin = index * absoluteBinSize + val endOfAbsoluteBin = (index+1) * absoluteBinSize + if (minDisplayRange <= startOfAbsoluteBin && endOfAbsoluteBin < maxDisplayRange) { + + val bin = SimpleHistogramBin( + startOfAbsoluteBin, + endOfAbsoluteBin, + true, + false + ) + bin.itemCount = longType.get().toInt() + max = max(bin.itemCount, max) + volumeHistogramData.addBin(bin) + } + } + return max + } + return null + } + /** - * Return a histogram over the set minDisplayRange and maxDisplayRange of the volumes viewState source (currently only using spimSource) + * Return a histogram over the whole volume ignoring the display range. Uses the volumes viewState source (currently only using spimSource). + * The function will select a miplevel which has less then [maximumResolution] voxels in side lengths and divide the results into [bins] + * different bins. */ - override fun generateHistogram(): Histogram1d<*>? { - var histogram : Histogram1d<*>? = null + private fun generateHistogramSPIMSourceOnCPU(maximumResolution: Int, bins: Int): Histogram1d<*>? { + val type = viewerState.sources.firstOrNull()?.spimSource?.type ?: return null + logger.info("Volume type is ${type.javaClass.simpleName}") + val context = if(volumeManager.hub?.getApplication()?.scijavaContext != null) { + volumeManager.hub?.getApplication()?.scijavaContext!! + } else { + Context(OpService::class.java) + } + val ops = context.getService(OpService::class.java) - this.viewerState.sources.firstOrNull()?.spimSource?.getSource(0, 0)?.let { rai -> - histogram = Histogram1d(Real1dBinMapper(minDisplayRange.toDouble(), maxDisplayRange.toDouble(), 1024, false)) - (histogram as Histogram1d).countData(rai as Iterable) + if(ops == null) { + logger.warn("Could not create OpService from scijava context, returning null histogram.") + return null } - return histogram + val miplevels = viewerState.sources.firstOrNull()?.spimSource?.numMipmapLevels ?: 0 + logger.info("Dataset has $miplevels miplevels") + + val reducedResolutionRAI = (0 until miplevels) + .map { it to viewerState.sources.first().spimSource.getSource(0, it) } + .firstOrNull { it.second.dimensionsAsLongArray().all { size -> size < maximumResolution } } + + val rai = if(reducedResolutionRAI == null) { + val r = viewerState.sources.first().spimSource.getSource(0, 0) + logger.info("Using default miplevel with dimensions ${r.dimensionsAsLongArray().joinToString("/")} for histogram calculation.") + r + } else { + logger.info("Using miplevel ${reducedResolutionRAI.first} with dimensions ${reducedResolutionRAI.second.dimensionsAsLongArray().joinToString("/")} for histogram calculation.") + reducedResolutionRAI.second + } + + val histogram = measureTimedValue { ops.run("image.histogram", rai, bins) as Histogram1d<*> } + + logger.info("Histogram creation took ${histogram.duration.inWholeMilliseconds}ms") + + return histogram.value } + private var slicingArray = FloatArray(4 * MAX_SUPPORTED_SLICING_PLANES) + /** * Returns array of slicing plane equations for planes assigned to this volume. */ fun slicingArray(): FloatArray { - if (slicingPlaneEquations.size > MAX_SUPPORTED_SLICING_PLANES) - logger.warn("More than ${MAX_SUPPORTED_SLICING_PLANES} slicing planes for ${this.name} set. Ignoring additional planes.") - - val fa = FloatArray(4 * MAX_SUPPORTED_SLICING_PLANES) + if (slicingPlaneEquations.size > MAX_SUPPORTED_SLICING_PLANES) { + logger.warn("More than $MAX_SUPPORTED_SLICING_PLANES slicing planes for ${this.name} set. Ignoring additional planes.") + } slicingPlaneEquations.entries.take(MAX_SUPPORTED_SLICING_PLANES).forEachIndexed { i, entry -> - fa[0+i*4] = entry.value.x - fa[1+i*4] = entry.value.y - fa[2+i*4] = entry.value.z - fa[3+i*4] = entry.value.w + slicingArray[0+i*4] = entry.value.x + slicingArray[1+i*4] = entry.value.y + slicingArray[2+i*4] = entry.value.z + slicingArray[3+i*4] = entry.value.w } - return fa + return slicingArray } /** @@ -544,7 +650,7 @@ open class Volume( companion object { val setupId = AtomicInteger(0) - val scifio: SCIFIO = SCIFIO() + lateinit var scifio: SCIFIO private val logger by lazyLogger() @JvmStatic @JvmOverloads fun fromSpimData( @@ -789,13 +895,62 @@ open class Volume( return buffer } + private fun readRawFile(path: Path, dimensions: Vector3i, bytesPerVoxel: Int, offsets: Pair? = null): ByteBuffer { + val buffer: ByteBuffer by lazy { + + val buffer = ByteArray(1024 * 1024) + val stream = FileInputStream(path.toFile()) + if(offsets != null) { + stream.skip(offsets.first) + } + + val imageData: ByteBuffer = MemoryUtil.memAlloc((bytesPerVoxel * dimensions.x * dimensions.y * dimensions.z)) + + logger.debug( + "{}: Allocated {} bytes for image of {} containing {} per voxel", + path.fileName, + imageData.capacity(), + dimensions, + bytesPerVoxel + ) + + val start = System.nanoTime() + var bytesRead = 0 + var total = 0 + while (true) { + var maxReadSize = minOf(buffer.size, imageData.capacity() - total) + maxReadSize = maxOf(maxReadSize, 1) + bytesRead = stream.read(buffer, 0, maxReadSize) + + if(bytesRead < 0) { + break + } + + imageData.put(buffer, 0, bytesRead) + + total += bytesRead + + if(offsets != null && total >= (offsets.second - offsets.first)) { + break + } + } + val duration = (System.nanoTime() - start) / 10e5 + logger.debug("Reading took $duration ms") + + imageData.flip() + imageData + } + + return buffer + } + /** * Reads a volume from the given [file]. */ @JvmStatic @JvmOverloads fun fromPath(file: Path, hub: Hub, onlyLoadFirst: Int? = null): BufferedVolume { if(file.normalize().toString().endsWith("raw")) { - return fromPathRaw(file, hub) + return fromPathRaw(file, hub, UnsignedByteType()) } var volumeFiles: List if(Files.isDirectory(file)) { @@ -812,6 +967,8 @@ open class Volume( volumeFiles = listOf(file) } + scifio = SCIFIO() + val volumes = CopyOnWriteArrayList() val dims = Vector3i() @@ -936,12 +1093,31 @@ open class Volume( } /** - * Reads raw volumetric data from a [file]. + * Reads raw volumetric data from a [file], assuming the input + * data is 16bit Unsigned Int. * * Returns the new volume. */ - @JvmStatic fun fromPathRaw(file: Path, hub: Hub): BufferedVolume { + @JvmStatic + fun > fromPathRaw( + file: Path, + hub: Hub + ): BufferedVolume { + return fromPathRaw(file, hub, UnsignedShortType()) + } + /** + * Reads raw volumetric data from a [file], with the [type] being + * explicitly specified. + * + * Returns the new volume. + */ + @JvmStatic + fun > fromPathRaw( + file: Path, + hub: Hub, + type: T + ): BufferedVolume { val infoFile: Path val volumeFiles: List @@ -963,32 +1139,75 @@ open class Volume( val volumes = CopyOnWriteArrayList() volumeFiles.forEach { v -> val id = v.fileName.toString() - val buffer: ByteBuffer by lazy { + logger.debug("Loading $id from disk") - logger.debug("Loading $id from disk") - val buffer = ByteArray(1024 * 1024) - val stream = FileInputStream(v.toFile()) - val imageData: ByteBuffer = MemoryUtil.memAlloc((2 * dimensions.x * dimensions.y * dimensions.z)) + val bytesPerVoxel = type.bitsPerPixel/8 + val buffer = readRawFile(v, dimensions, bytesPerVoxel) - logger.debug("${v.fileName}: Allocated ${imageData.capacity()} bytes for UINT16 image of $dimensions") + volumes.add(BufferedVolume.Timepoint(id, buffer)) + } - val start = System.nanoTime() - var bytesRead = stream.read(buffer, 0, buffer.size) - while (bytesRead > -1) { - imageData.put(buffer, 0, bytesRead) - bytesRead = stream.read(buffer, 0, buffer.size) - } - val duration = (System.nanoTime() - start) / 10e5 - logger.debug("Reading took $duration ms") + return fromBuffer(volumes, dimensions.x, dimensions.y, dimensions.z, type, hub) + } - imageData.flip() - imageData + /** + * Reads raw volumetric data from a [file], splits it into buffers of at most, and as close as possible to, + * [sizeLimit] bytes and creates a volume from each buffer. + * + * Returns the list of volumes. + */ + @JvmStatic + fun > fromPathRawSplit( + file: Path, + type: T, + sizeLimit: Long = 2000000000L, + hub: Hub + ): Pair> { + + val infoFile = file.resolveSibling("stacks.info") + + val lines = Files.lines(infoFile).toList() + + logger.debug("reading stacks.info (${lines.joinToString()}) (${lines.size} lines)") + val dimensions = Vector3i(lines.get(0).split(",").map { it.toInt() }.toIntArray()) + val bytesPerVoxel = type.bitsPerPixel/8 + + var slicesRemaining = dimensions.z + var bytesRead = 0L + var numPartitions = 0 + + val slicesPerPartition = floor(sizeLimit.toFloat()/(bytesPerVoxel * dimensions.x * dimensions.y)).toInt() + + val children = ArrayList() + + while (slicesRemaining > 0) { + val slices = if(slicesRemaining > slicesPerPartition) { + slicesPerPartition + } else { + slicesRemaining } - volumes.add(BufferedVolume.Timepoint(id, buffer)) + val partitionDims = Vector3i(dimensions.x, dimensions.y, slices) + val size = bytesPerVoxel * dimensions.x * dimensions.y * slices + + val window = bytesRead to bytesRead+size-1 + + logger.debug("Reading raw file with offsets: $window") + val buffer = readRawFile(file, partitionDims, bytesPerVoxel, window) + + val volume = ArrayList() + volume.add(BufferedVolume.Timepoint(file.fileName.toString(), buffer)) + children.add(fromBuffer(volume, partitionDims.x, partitionDims.y, partitionDims.z, type, hub)) + + slicesRemaining -= slices + numPartitions += 1 + bytesRead += size } - return fromBuffer(volumes, dimensions.x, dimensions.y, dimensions.z, UnsignedShortType(), hub) + val parent = RichNode() + children.forEach { parent.addChild(it) } + + return parent to children } /** Amount of supported slicing planes per volume, see also sampling shader segments */ @@ -1015,3 +1234,4 @@ open class Volume( } } + diff --git a/src/main/kotlin/graphics/scenery/volumes/VolumeHistogramComputeNode.kt b/src/main/kotlin/graphics/scenery/volumes/VolumeHistogramComputeNode.kt new file mode 100644 index 0000000000..ec15bd97bc --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/VolumeHistogramComputeNode.kt @@ -0,0 +1,157 @@ +package graphics.scenery.volumes + +import graphics.scenery.RichNode +import graphics.scenery.Scene +import graphics.scenery.ShaderMaterial +import graphics.scenery.ShaderProperty +import graphics.scenery.backends.Renderer +import graphics.scenery.compute.ComputeMetadata +import graphics.scenery.compute.InvocationType +import graphics.scenery.textures.Texture +import kotlinx.coroutines.runBlocking +import net.imglib2.type.numeric.integer.IntType +import net.imglib2.type.numeric.integer.UnsignedByteType +import net.imglib2.type.numeric.integer.UnsignedShortType +import org.joml.Vector3i +import org.lwjgl.system.MemoryUtil +import java.nio.ByteBuffer +import java.nio.IntBuffer + +/** + * A compute node to calculate a histogram of a volume on the gpu. To use call [VolumeHistogramComputeNode.generateHistogram]. + * + * @author Aryaman Gupta + * @author Jan Tiemann + */ +class VolumeHistogramComputeNode(val volume: Volume, data: ByteBuffer): RichNode() { + + var dataType = UnsignedByteType() + private val buffer: ByteBuffer + + val histogramTexture: Texture + var result: Texture? = null + + @ShaderProperty + var numBins = 100 + + @ShaderProperty + val volumeIs8Bit: Boolean = when(volume.bytesPerVoxel){ + 1 -> true + 2 -> false + else -> throw IllegalArgumentException("only 8 and 16 bit data supported for histograms") + } + + @ShaderProperty + val numVoxels: Int + + @ShaderProperty + val maxDisplayVal: Float + + @ShaderProperty + val minDisplayVal: Float + + init { + this.name = "compute node" + + numVoxels = volume.getDimensions().z + + maxDisplayVal = volume.maxDisplayRange + minDisplayVal = volume.minDisplayRange + + buffer = MemoryUtil.memCalloc(numBins * 4 * 2 * 2) + + histogramTexture = Texture( + Vector3i(numBins, 2, 2), + 1, + contents = buffer, + usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture + ), + type = IntType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + + + + this.setMaterial(ShaderMaterial.fromFiles(this::class.java, "ComputeHistogram.comp")) { + textures["Volume8Bit"] = if (volumeIs8Bit) { + Texture( + Vector3i(volume.getDimensions()), + 1, + contents = data, + usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + type = dataType, + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + } else { + Texture() + } + + + textures["Volume16Bit"] = if (!volumeIs8Bit) { + Texture( + Vector3i(volume.getDimensions()), + 1, + contents = data, + usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + type = UnsignedShortType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + } else { + Texture() + } + + + textures["Histogram"] = histogramTexture + } + + this.metadata["ComputeMetadata"] = ComputeMetadata( + workSizes = Vector3i(volume.getDimensions().x, volume.getDimensions().y, 1), + invocationType = InvocationType.Once + ) + + } + + /** + * waits a bit and then fetches the histogram from the node. + * @return list of number of items in the histogram bins, sorted by ascending bin label + */ + fun fetchHistogram(scene: Scene, renderer: Renderer): List { + + var buf: IntBuffer? = null + Thread.sleep(500) // dunno why this is required. Probably some scenery update stuff + runBlocking { + renderer.requestTexture(histogramTexture) { + buf = it.contents!!.asIntBuffer() + }.join() + } + scene.removeChild(this) + + val list = mutableListOf() + buf?.limit(numBins) + buf?.let { + while (it.hasRemaining()){ + list.add(it.get()) + } + } + + return list + } + + companion object { + /** + * Creates and attaches a histogram compute node to the scene. To get the histogram call [fetchHistogram] + */ + fun generateHistogram(volume: Volume, data: ByteBuffer, scene: Scene): VolumeHistogramComputeNode { + val volumeHistogram = VolumeHistogramComputeNode(volume, data) + scene.addChild(volumeHistogram) + return volumeHistogram + } + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt index 86a80edbaa..c7ec601e6d 100644 --- a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt +++ b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt @@ -9,13 +9,13 @@ import bdv.viewer.state.SourceState import bvv.core.backend.Texture import bvv.core.backend.Texture3D import bvv.core.cache.* -import bvv.core.render.VolumeBlocks -import bvv.core.render.VolumeShaderSignature import bvv.core.multires.MultiResolutionStack3D import bvv.core.multires.SimpleStack3D import bvv.core.multires.SourceStacks import bvv.core.multires.Stack3D import bvv.core.render.MultiVolumeShaderMip +import bvv.core.render.VolumeBlocks +import bvv.core.render.VolumeShaderSignature import bvv.core.shadergen.generate.Segment import bvv.core.shadergen.generate.SegmentTemplate import bvv.core.shadergen.generate.SegmentType @@ -44,7 +44,6 @@ import java.nio.IntBuffer import java.util.* import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.ForkJoinPool -import java.util.function.BiConsumer import kotlin.math.max import kotlin.math.min import kotlin.system.measureTimeMillis @@ -59,7 +58,7 @@ class VolumeManager( override var hub: Hub?, val useCompute: Boolean = false, val customSegments: Map? = null, - val customBindings: BiConsumer, Map>? = null + val customBindings: MultiVolumeShaderMip.SegmentConsumer? = null ) : DefaultNode("VolumeManager"), HasGeometry, HasRenderable, HasMaterial, Hubable, RequestRepaint { /** @@ -130,7 +129,7 @@ class VolumeManager( /** List of custom-created textures not to be cleared automatically */ var customTextures = arrayListOf() - + var customUniforms = arrayListOf() init { addRenderable { state = State.Created @@ -171,7 +170,10 @@ class VolumeManager( @Synchronized private fun recreateMaterial(context: SceneryContext) { - shaderProperties.clear() + val oldProperties = shaderProperties.filter { it.key !in customUniforms }.keys + oldProperties.forEach{ + shaderProperties.remove(it) + } shaderProperties["transform"] = Matrix4f() shaderProperties["viewportSize"] = Vector2f() shaderProperties["dsp"] = Vector2f() @@ -264,7 +266,7 @@ class VolumeManager( segments[SegmentType.FragmentShader] = SegmentTemplate( this.javaClass, "BDVVolume.frag", - "intersectBoundingBox", "vis", "SampleVolume", "Convert", "Accumulate" + "intersectBoundingBox", "vis", "localNear", "localFar", "SampleVolume", "Convert", "Accumulate" ) segments[SegmentType.MaxDepth] = SegmentTemplate( this.javaClass, @@ -302,23 +304,70 @@ class VolumeManager( ) segments[SegmentType.AccumulatorMultiresolution] = SegmentTemplate( "AccumulateBlockVolume.frag", - "vis", "sampleVolume", "convert", "sceneGraphVisibility" + "vis", "localNear", "localFar", "sampleVolume", "convert", "sceneGraphVisibility" ) segments[SegmentType.Accumulator] = SegmentTemplate( "AccumulateSimpleVolume.frag", - "vis", "sampleVolume", "convert", "sceneGraphVisibility" + "vis", "localNear", "localFar", "sampleVolume", "convert", "sceneGraphVisibility" ) customSegments?.forEach { (type, segment) -> segments[type] = segment } + var triggered = false val additionalBindings = customBindings - ?: BiConsumer { _: Map, instances: Map -> + ?: MultiVolumeShaderMip.SegmentConsumer { _: Map, + segmentInstances: Map, + volumeIndex: Int -> logger.debug("Connecting additional bindings") - instances[SegmentType.SampleMultiresolutionVolume]?.bind("convert", instances[SegmentType.Convert]) - instances[SegmentType.SampleVolume]?.bind("convert", instances[SegmentType.Convert]) - instances[SegmentType.SampleVolume]?.bind("sceneGraphVisibility", instances[SegmentType.Accumulator]) - instances[SegmentType.SampleMultiresolutionVolume]?.bind("sceneGraphVisibility", instances[SegmentType.AccumulatorMultiresolution]) + if(!triggered) { + segmentInstances[SegmentType.FragmentShader]?.repeat("localNear", n) + segmentInstances[SegmentType.FragmentShader]?.repeat("localFar", n) + triggered = true + } + + if(signatures[volumeIndex].sourceStackType == SourceStacks.SourceStackType.MULTIRESOLUTION) { + segmentInstances[SegmentType.FragmentShader]?.bind( + "localNear", + volumeIndex, + segmentInstances[SegmentType.AccumulatorMultiresolution] + ) + segmentInstances[SegmentType.FragmentShader]?.bind( + "localFar", + volumeIndex, + segmentInstances[SegmentType.AccumulatorMultiresolution] + ) + } else { + segmentInstances[SegmentType.FragmentShader]?.bind( + "localNear", + volumeIndex, + segmentInstances[SegmentType.Accumulator] + ) + segmentInstances[SegmentType.FragmentShader]?.bind( + "localFar", + volumeIndex, + segmentInstances[SegmentType.Accumulator] + ) + } + + segmentInstances[SegmentType.SampleMultiresolutionVolume]?.bind( + "convert", + segmentInstances[SegmentType.Convert] + ) + segmentInstances[SegmentType.SampleVolume]?.bind( + "convert", + segmentInstances[SegmentType.Convert] + ) + + segmentInstances[SegmentType.SampleVolume]?.bind( + "sceneGraphVisibility", + segmentInstances[SegmentType.Accumulator] + ) + segmentInstances[SegmentType.SampleMultiresolutionVolume]?.bind( + "sceneGraphVisibility", + segmentInstances[SegmentType.AccumulatorMultiresolution] + ) + } val newProgvol = MultiVolumeShaderMip( @@ -774,6 +823,31 @@ class VolumeManager( hub?.add(vm) } + /** + * Replaces the VolumeManager [toReplace] with the current VolumeManager, transferring volumes to the + * current VolumeManager. All other properties, e.g., [customSegments], [customTextures], etc. of both + * VolumeManagers remain unmodified. + */ + fun replace(toReplace: VolumeManager) { + logger.debug("Replacing volume manager with ${toReplace.nodes.size} volumes managed") + + hub?.remove(toReplace) + + //remove the volumes currently held by this volume manager + nodes.forEach { + remove(it) + } + + //add the volumes held by the volume manager to be replaced into this volume manager + val volumes = toReplace.nodes.toMutableList() + volumes.forEach { + add(it) + it.volumeManager = this + } + + hub?.add(this) + } + @Synchronized fun remove(node: Volume) { logger.debug("Removing $node to OOC nodes") diff --git a/src/main/kotlin/graphics/scenery/volumes/vdi/VDIData.kt b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIData.kt new file mode 100644 index 0000000000..97891d6735 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIData.kt @@ -0,0 +1,56 @@ +package graphics.scenery.volumes.vdi + +import org.jetbrains.annotations.ApiStatus.Experimental +import org.joml.Matrix4f +import org.joml.Vector2i +import org.joml.Vector3f + +/** + * Defines the metadata that is required for reading and rendering a Volumetric Depth Image (VDI). + * + * [version] The version of the VDI generation code. Used to prevent errors due to code version mismatch. + * [index] The index of the VDI streamed in an ongoing VDI streaming session + * [projection] The projection matrix of the camera used to generate the VDI. + * [view] The view matrix of the camera used to generate the VDI. + * [model] The model matrix of the volume in the scene for which the VDI was generated. + * [volumeDimensions] The dimensions (in voxels) of the volume on which the VDI was generated. + * [windowDimensions] The display resolution for which the VDI was generated. + * [nw] Parameter from BigDataViewer which defines voxel length in world space. + * + * @author Aryaman Gupta and Ulrik Günther + */ +@Experimental +data class VDIMetadata( + val version: Int = 1, + var index: Int = 0, + var projection: Matrix4f = Matrix4f(), + var view: Matrix4f = Matrix4f(), + var model: Matrix4f = Matrix4f(), + val volumeDimensions: Vector3f = Vector3f(), + var windowDimensions: Vector2i = Vector2i(), + var nw: Float = 0f +) + +/** + * The sizes (in bytes), after potential compression, of the VDI buffers generated for streaming or storage. + * + * [colorSize] The size of the buffer containing colors of the supersegments contained in the VDI. + * [depthSize] The size of the buffer containing front and back depths of the supersegments contained in the VDI. + * [accelGridSize] The size of the acceleration data structure generated on the VDI. + */ +data class VDIBufferSizes( + var colorSize: Long = 0, + var depthSize: Long = 0, + var accelGridSize: Long = 0 +) + +/** + * The buffer sizes ([VDIBufferSizes]) and the metadata ([VDIMetadata]) corresponding to the generated VDI. + * + * [bufferSizes] The sizes (in bytes), after potential compression, of the VDI buffers. + * [metadata] The metadata that is required for reading and rendering a VDI. + */ +data class VDIData( + val bufferSizes: VDIBufferSizes = VDIBufferSizes(), + val metadata: VDIMetadata = VDIMetadata() +) diff --git a/src/main/kotlin/graphics/scenery/volumes/vdi/VDIDataIO.kt b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIDataIO.kt new file mode 100644 index 0000000000..6d1204d83c --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIDataIO.kt @@ -0,0 +1,76 @@ +package graphics.scenery.volumes.vdi + +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output +import org.jetbrains.annotations.ApiStatus.Experimental +import org.joml.Matrix4f +import org.joml.Vector2i +import org.joml.Vector3f +import java.io.InputStream +import java.io.OutputStream + +/** + * A utility class handling reading and writing of metadata for Volumetric Depth + * Images (VDIs) ([VDIData]) with serialization handled by [Kryo]. + * + * @author Aryaman Gupta and Ulrik Günther + */ +@Experimental +class VDIDataIO { + companion object { + + private fun freeze(kryo: Kryo?): Kryo { + return if(kryo == null) { + val temp = Kryo() + temp.register(VDIData::class.java) + temp.register(VDIBufferSizes::class.java) + temp.register(VDIMetadata::class.java) + temp.register(Matrix4f::class.java) + temp.register(Vector3f::class.java) + temp.register(Vector2i::class.java) + temp + } else { + kryo + } + } + + /** + * Reads [VDIData] from a serialized [InputStream]. + * + * @param[from] The serialized [InputStream] from which the [VDIData] is to be read. + * @param[kryo] A [Kryo] instance with all required classes registered (see [freeze]). If null, + * a new [Kryo] will be instantiated and registered with required classes using [freeze]. + * + * @return The deserialized [VDIData]. + */ + @JvmStatic + fun read(from: InputStream, kryo: Kryo? = null): VDIData { + val k = freeze(kryo) + val input = Input(from) + val read = k.readClassAndObject(input) + input.close() + return read as VDIData + } + + /** + * Writes [VDIData] to a serialized [OutputStream]. + * + * @param[vdiData] The [VDIData] to be serialized. + * @param[to] The [OutputStream] to which the serialized [vdiData] is to be written. + * @param[kryo] A [Kryo] instance with all required classes registered (see [freeze]). If null, + * a new [Kryo] will be instantiated and registered with required classes using [freeze]. + * + * @return The total number of bytes written to the [OutputStream]. + */ + @JvmStatic + fun write(vdiData: VDIData, to: OutputStream, kryo: Kryo? = null): Long { + val k = freeze(kryo) + val output = Output(to) + k.writeClassAndObject(output, vdiData) + val total = output.total() + output.close() + return total + } + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/vdi/VDINode.kt b/src/main/kotlin/graphics/scenery/volumes/vdi/VDINode.kt new file mode 100644 index 0000000000..eae9b95575 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/vdi/VDINode.kt @@ -0,0 +1,335 @@ +package graphics.scenery.volumes.vdi + +import graphics.scenery.RichNode +import graphics.scenery.ShaderMaterial +import graphics.scenery.ShaderProperty +import graphics.scenery.backends.Shaders +import graphics.scenery.compute.ComputeMetadata +import graphics.scenery.compute.InvocationType +import graphics.scenery.textures.Texture +import graphics.scenery.textures.UpdatableTexture +import graphics.scenery.utils.Image +import graphics.scenery.utils.extensions.applyVulkanCoordinateSystem +import net.imglib2.type.numeric.integer.UnsignedIntType +import net.imglib2.type.numeric.real.FloatType +import org.jetbrains.annotations.ApiStatus.Experimental +import org.joml.Matrix4f +import org.joml.Vector3f +import org.joml.Vector3i +import org.lwjgl.system.MemoryUtil +import java.nio.ByteBuffer + +/** + * A class defining the properties and textures required to render a Volumetric Depth Image (VDI). Rendering a VDI requires generating + * a VDINode and adding it to the scene. The class provides public functions to update the VDI being rendered and its properties. Double + * buffering is supported so that VDIs can be updated without interrupting the rendering. + * + * @param[windowWidth] The width (resolution in pixels along x-axis) of the current rendering window. + * @param[windowHeight] The height (resolution in pixels along y-axis) of the current rendering window. + * @param[numSupersegments] The maximum number of supersegments along any ray/pixel of the VDI, i.e., the z-dimension of the VDI. + * @param[vdiData] The metadata ([VDIData]) associated with the VDI. + */ + +@Experimental +class VDINode(windowWidth: Int, windowHeight: Int, val numSupersegments: Int, vdiData: VDIData) : RichNode() { + + /** The projection matrix of the camera viewport that was used to generate the VDI in the first (default) buffer */ + @ShaderProperty + private var ProjectionOriginal = Matrix4f() + + /** The inverse of the projection matrix of the camera viewport that was used to generate the VDI in the first (default) buffer */ + @ShaderProperty + private var invProjectionOriginal = Matrix4f() + + /** The view matrix of the camera viewport that was used to generate the VDI in the first (default) buffer */ + @ShaderProperty + private var ViewOriginal = Matrix4f() + + /** The inverse of the view matrix of the camera viewport that was used to generate the VDI in the first (default) buffer */ + @ShaderProperty + private var invViewOriginal = Matrix4f() + + /** The view matrix of the camera viewport that was used to generate the VDI in the second buffer */ + @ShaderProperty + private var ViewOriginal2 = Matrix4f() + + /** The inverse of the view matrix of the camera viewport that was used to generate the VDI in the second buffer */ + @ShaderProperty + private var invViewOriginal2 = Matrix4f() + + /** Indicates whether the second buffer should be used for rendering. The first buffer is used if false. */ + @ShaderProperty + private var useSecondBuffer = false + + /** Inverse of the model matrix of the volume in the scene on which the VDI was generated. */ + @ShaderProperty + private var invModel = Matrix4f() + + /** The dimensions (in voxels, x, y and z) of the volume on which the VDI was generated. */ + @ShaderProperty + private var volumeDims = Vector3f() + + /** BigVolumeViewer property storing the voxel side length world space of the volume on which the VDI was generated. */ + @ShaderProperty + private var nw = 0f + + /** The total supersegments in the entire VDI. */ + @ShaderProperty + private var totalGeneratedSupsegs: Int = 0 + + /** Should the rendering be subsampled along the ray? Subsampling leads to better performance but worse rendering quality. */ + @ShaderProperty + private var do_subsample = false + + /** The maximum permitted samples along the ray. Only relevant if [do_subsample] is true. */ + @ShaderProperty + private var max_samples = 50 + + /** The sampling factor along the ray. Only relevant if [do_subsample] is true. Larger value leads to better quality at the cost of performance */ + @ShaderProperty + private var sampling_factor = 0.1f + + /** Controls the downsampling of the display resolution. Lower values accelerate rendering at the cost of quality. Value 1.0f represents full resolution rendering. */ + @ShaderProperty + private var downImage = 1f + + /** The rendering window width (x-axis of the display resolution) for which the VDI was generated. */ + @ShaderProperty + var vdiWidth: Int = 0 + + /** The rendering window height (y-axis of the display resolution) for which the VDI was generated. */ + @ShaderProperty + var vdiHeight: Int = 0 + + /** Whether empty regions should be skipped or not. Accelerates rendering without loss of quality. */ + @ShaderProperty + var skip_empty = true + + /** + * Enum class recording which of the two VDIs is currently being rendered. + * + * VDI rendering supports double buffering on the GPU so that one VDI can be updated while the other is being rendered. + */ + enum class DoubleBuffer { + First, + Second + } + + /** An object of the enum class [DoubleBuffer]. Tracks which buffer is currently being rendered. */ + private var currentBuffer = DoubleBuffer.First + + var wantsSync = false + override fun wantsSync(): Boolean = wantsSync + + init { + name = "vdi node" + setMaterial(ShaderMaterial(Shaders.ShadersFromFiles(arrayOf("RaycastVDI.comp"), this@VDINode::class.java))) + + val opBuffer = MemoryUtil.memCalloc(windowWidth * windowHeight * 4) + + material().textures["OutputViewport"] = Texture.fromImage( + Image(opBuffer, windowWidth, windowHeight), + usage = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + metadata["ComputeMetadata"] = ComputeMetadata( + workSizes = Vector3i(windowWidth, windowHeight, 1), + invocationType = InvocationType.Permanent + ) + visible = true + + updateMetadata(vdiData) + } + + /** + * Returns the dimensions of the acceleration grid data structure of the VDI. These dimensions depend on the resolution of the VDI. When + * a parameter [vdiData] is supplied, the resolution of the VDI is inferred based on the metadata contained within. When the parameter is + * not provided, the resolution of the VDI represented by this object is used. + * + * @param[vdiData] Optional parameter containing the metadata ([VDIData]) associated with the VDI to which the acceleration grid belongs. + * + * @return The dimensions of the acceleration grid. + */ + fun getAccelerationGridSize(vdiData: VDIData? = null) : Vector3f { + return if(vdiData == null) { + Vector3f(vdiWidth/8f, vdiHeight/8f, numSupersegments.toFloat()) + } else { + Vector3f(vdiData.metadata.windowDimensions.x/8f, vdiData.metadata.windowDimensions.y/8f, numSupersegments.toFloat()) + } + } + + /** + * Attaches textures containing the VDI data for rendering. + * + * @param[colBuffer] A [ByteBuffer] containing the colors of the supsersegments in the VDI + * @param[depthBuffer] A [ByteBuffer] containing the depths of the supsersegments in the VDI + * @param[gridBuffer] A [ByteBuffer] containing the acceleration grid for the VDI + * @param[toBuffer] Defines which of the VDI buffers in the double-buffering system the textures should be attached to. + * Defaults to [DoubleBuffer.First] + */ + fun attachTextures(colBuffer: ByteBuffer, depthBuffer: ByteBuffer, gridBuffer: ByteBuffer, toBuffer: DoubleBuffer = DoubleBuffer.First) { + + val numGridCells = getAccelerationGridSize() + + if(toBuffer == DoubleBuffer.First) { + material().textures[inputColorTexture] = Texture(Vector3i(numSupersegments, vdiHeight, vdiWidth), 4, contents = colBuffer, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture) + , type = FloatType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + material().textures[inputDepthTexture] = Texture(Vector3i(2 * numSupersegments, vdiHeight, vdiWidth), channels = 1, contents = depthBuffer, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), type = FloatType(), mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + material().textures[inputAccelerationTexture] = Texture(Vector3i(numGridCells.x.toInt(), numGridCells.y.toInt(), numGridCells.z.toInt()), 1, type = UnsignedIntType(), contents = gridBuffer, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + } else { + material().textures["${inputColorTexture}2"] = Texture(Vector3i(numSupersegments, vdiHeight, vdiWidth), 4, contents = colBuffer, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture) + , type = FloatType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + material().textures["${inputDepthTexture}2"] = Texture(Vector3i(2 * numSupersegments, vdiHeight, vdiWidth), channels = 1, contents = depthBuffer, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), type = FloatType(), mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + material().textures["${inputAccelerationTexture}2"] = Texture(Vector3i(numGridCells.x.toInt(), numGridCells.y.toInt(), numGridCells.z.toInt()), 1, type = UnsignedIntType(), contents = gridBuffer, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + } + } + + /** + * Attaches empty textures so that the VDI node can be placed into the scene before contents of a VDI are available + * (e.g. in streaming applications). + * + * @param[toBuffer] Defines which of the VDI buffers in the double-buffering system the textures should be attached to. + */ + fun attachEmptyTextures(toBuffer: DoubleBuffer) { + val emptyColor = MemoryUtil.memCalloc(4 * 4) + val emptyColorTexture = Texture(Vector3i(1, 1, 1), 4, contents = emptyColor, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + type = FloatType(), mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + val emptyDepth = MemoryUtil.memCalloc(1 * 4) + val emptyDepthTexture = Texture(Vector3i(1, 1, 1), 1, contents = emptyDepth, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + type = FloatType(), mipmap = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + val emptyAccel = MemoryUtil.memCalloc(4) + val emptyAccelTexture = Texture( + Vector3i(1, 1, 1), 1, contents = emptyAccel, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + type = UnsignedIntType(), mipmap = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour + ) + + if (toBuffer == DoubleBuffer.First ) { + material().textures[inputColorTexture] = emptyColorTexture + material().textures[inputDepthTexture] = emptyDepthTexture + material().textures[inputAccelerationTexture] = emptyAccelTexture + } else { + material().textures["${inputColorTexture}2"] = emptyColorTexture + material().textures["${inputDepthTexture}2"] = emptyDepthTexture + material().textures["${inputAccelerationTexture}2"] = emptyAccelTexture + } + } + + /** + * Asynchronously updates the VDI on the GPU using double buffering and [UpdatableTexture]s. + */ + private fun updateTextures(color: ByteBuffer, depth: ByteBuffer, accelGridBuffer: ByteBuffer) { + + val colorTexture = UpdatableTexture(Vector3i(numSupersegments, vdiHeight, vdiWidth), 4, contents = null, usageType = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture, Texture.UsageType.AsyncLoad), + type = FloatType(), mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + val colorUpdate = UpdatableTexture.TextureUpdate( + UpdatableTexture.TextureExtents(0, 0, 0, numSupersegments, vdiHeight, vdiWidth), + color.slice() + ) + colorTexture.addUpdate(colorUpdate) + + + val depthTexture = UpdatableTexture(Vector3i(2 * numSupersegments, vdiHeight, vdiWidth), channels = 1, contents = null, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture, Texture.UsageType.AsyncLoad), type = FloatType(), mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + val depthUpdate = UpdatableTexture.TextureUpdate( + UpdatableTexture.TextureExtents(0, 0, 0, 2 * numSupersegments, vdiHeight, vdiWidth), + depth.slice() + ) + depthTexture.addUpdate(depthUpdate) + + + val numGridCells = getAccelerationGridSize() + + val accelTexture = UpdatableTexture(Vector3i(numGridCells.x.toInt(), numGridCells.y.toInt(), numGridCells.z.toInt()), channels = 1, contents = null, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture, Texture.UsageType.AsyncLoad), type = UnsignedIntType(), mipmap = false, normalized = true, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + + val accelUpdate = UpdatableTexture.TextureUpdate( + UpdatableTexture.TextureExtents(0, 0, 0, vdiWidth / 8, vdiHeight / 8, numSupersegments), + accelGridBuffer + ) + accelTexture.addUpdate(accelUpdate) + + + if(currentBuffer == DoubleBuffer.First) { + material().textures[inputColorTexture] = colorTexture + material().textures[inputDepthTexture] = depthTexture + material().textures[inputAccelerationTexture] = accelTexture + } else { + material().textures["${inputColorTexture}2"] = colorTexture + material().textures["${inputDepthTexture}2"] = depthTexture + material().textures["${inputAccelerationTexture}2"] = accelTexture + } + + while (!colorTexture.availableOnGPU() || !depthTexture.availableOnGPU() || !accelTexture.availableOnGPU()) { + logger.debug("Waiting for texture transfer. color: ${colorTexture.availableOnGPU()}, depth: ${depthTexture.availableOnGPU()}, grid: ${accelTexture.availableOnGPU()}") + Thread.sleep(10) + } + + logger.debug("Data has been detected to be uploaded to GPU") + } + + private fun updateMetadata(vdiData: VDIData) { + this.ProjectionOriginal = Matrix4f(vdiData.metadata.projection).applyVulkanCoordinateSystem() + this.invProjectionOriginal = Matrix4f(vdiData.metadata.projection).applyVulkanCoordinateSystem().invert() + this.nw = vdiData.metadata.nw + this.vdiWidth = vdiData.metadata.windowDimensions.x + this.vdiHeight = vdiData.metadata.windowDimensions.y + this.invModel = Matrix4f(vdiData.metadata.model).invert() + this.volumeDims = vdiData.metadata.volumeDimensions + + if(currentBuffer == DoubleBuffer.First) { + this.ViewOriginal = vdiData.metadata.view + this.invViewOriginal = Matrix4f(vdiData.metadata.view).invert() + } else { + this.ViewOriginal2 = vdiData.metadata.view + this.invViewOriginal2 = Matrix4f(vdiData.metadata.view).invert() + } + } + + /** + * Update the VDI currently being rendered with a new one. The contents of the new VDI are to be provided in the form of + * [UpdatableTexture]s. + * + * The function transparently handles double buffering - the new VDI is uploaded to the GPU in a different buffer than the + * VDI currently being rendered. The rendering buffer is switched once the upload is complete. The function returns after the + * upload is complete. + * + * @param[vdiData] The metadata ([VDIData]) associated with the new VDI + * @param[color] A [ByteBuffer] containing the colors of the supersegments in the new VDI + * @param[depth] A [ByteBuffer] containing the depths of the supersegments in the new VDI + * @param[accelGridBuffer] A [ByteBuffer] containing the grid acceleration data structure for the new VDI + */ + fun updateVDI(vdiData: VDIData, color: ByteBuffer, depth: ByteBuffer, accelGridBuffer: ByteBuffer) { + updateMetadata(vdiData) + updateTextures(color, depth, accelGridBuffer) + + if(currentBuffer == DoubleBuffer.First) { + useSecondBuffer = false + //The next buffer to which data should be uploaded is the second one + currentBuffer = DoubleBuffer.Second + } else { + useSecondBuffer = true + //The next buffer to which data should be uploaded is the first one + currentBuffer = DoubleBuffer.First + } + } + + companion object { + const val inputColorTexture = "InputVDI" + const val inputDepthTexture = "DepthVDI" + const val inputAccelerationTexture = "AccelerationGrid" + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/vdi/VDIStreamer.kt b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIStreamer.kt new file mode 100644 index 0000000000..2357875e63 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIStreamer.kt @@ -0,0 +1,374 @@ +package graphics.scenery.volumes.vdi + +import graphics.scenery.Camera +import graphics.scenery.Settings +import graphics.scenery.backends.Renderer +import graphics.scenery.utils.DataCompressor +import graphics.scenery.utils.extensions.fetchFromGPU +import graphics.scenery.utils.lazyLogger +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.VolumeManager +import org.jetbrains.annotations.ApiStatus.Experimental +import org.joml.Vector2i +import org.joml.Vector3f +import org.lwjgl.system.MemoryUtil +import org.zeromq.SocketType +import org.zeromq.ZContext +import org.zeromq.ZMQ +import org.zeromq.ZMQException +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import kotlin.system.measureNanoTime + +/** + * Class to support streaming of Volumetric Depth Images (VDIs). Provides public functions to stream generated VDIs on the + * server side and to receive and update them on the client side. + */ +@Experimental +class VDIStreamer { + + private val logger by lazyLogger() + + /** param to determine the state of vdi streaming */ + var vdiStreaming: AtomicBoolean = AtomicBoolean(false) + + /** the number of VDIs streamed so far */ + private var vdisStreamed: Int = 0 + + /** is this the first VDI received so far? */ + private var firstVDIReceived = true + + /** the ZMQ context with 4 threads used for publishing the VDI */ + private val context: ZContext = ZContext(4) + + private fun createPublisher(context: ZContext, address : String) : ZMQ.Socket { + val publisher: ZMQ.Socket = context.createSocket(SocketType.PUB) + publisher.isConflate = true + + try { + logger.warn(address) + publisher.bind(address) + address.substringAfterLast(":").toInt() + } catch (e: ZMQException) { + logger.warn("Binding failed, trying random port: $e") + publisher.bindToRandomPort(address.substringBeforeLast(":")) + } + return publisher + } + + /** + * Sets up streaming of VDIs to a chosen [ipAddress]. + * + * @param[ipAddress] The network address (IP address and port number) to which the VDIs should be streamed + * @param[cam] The camera that is generating the VDI + * @param[volumeDim] The dimensions of [volume] on which the VDIs are being generated + * @param[volume] The volume on which VDIs are being generated + * @param[maxSupersegments] The maximum number of supersegments in any list of the VDI, i.e., its resolution along z + * @param[vdiVolumeManager] The [VolumeManager] set up to generate VDIs + * @param[renderer] The renderer for this application + */ + fun setup( + ipAddress: String, + cam: Camera, + volumeDim: Vector3f, + volume: Volume, + maxSupersegments: Int, + vdiVolumeManager: VolumeManager, + renderer: Renderer + ) { + + val vdiData = VDIData( + VDIBufferSizes(), + VDIMetadata( + volumeDimensions = volumeDim, + ) + ) + + var firstFrame = true + + val publisher = createPublisher(context, ipAddress) + + var compressedColor: ByteBuffer? = null + var compressedDepth: ByteBuffer? = null + + val compressionTool = DataCompressor.CompressionTool.LZ4 + val compressor = DataCompressor(compressionTool) + + var vdiColorBuffer: ByteBuffer? + var vdiDepthBuffer: ByteBuffer? + var gridCellsBuff: ByteBuffer? + + val vdiColor = vdiVolumeManager.material().textures[VDIVolumeManager.colorTextureName]!! + + val vdiDepth = vdiVolumeManager.material().textures[VDIVolumeManager.depthTextureName]!! + + val gridCells = vdiVolumeManager.material().textures[VDIVolumeManager.accelerationTextureName]!! + + renderer.runAfterRendering.add { + + if (!firstFrame && vdiStreaming.get()) { + + vdiColor.fetchFromGPU() + vdiDepth.fetchFromGPU() + gridCells.fetchFromGPU() + + val model = volume.spatial().world + + vdiData.metadata.model = model + vdiData.metadata.index = vdisStreamed + vdiData.metadata.projection = cam.spatial().projection + vdiData.metadata.view = cam.spatial().getTransformation() + vdiData.metadata.windowDimensions = Vector2i(cam.width, cam.height) + vdiData.metadata.nw = vdiVolumeManager.shaderProperties["nw"] as Float + + vdiColorBuffer = vdiColor.contents + vdiDepthBuffer = vdiDepth.contents + gridCellsBuff = gridCells.contents + + val colorSize = cam.height * cam.width * maxSupersegments * 4 * 4 + val depthSize = cam.width * cam.height * maxSupersegments * 4 * 2 + val accelSize = (cam.width / 8) * (cam.height / 8) * maxSupersegments * 4 + + if (vdiColorBuffer!!.remaining() != colorSize || vdiDepthBuffer!!.remaining() != depthSize || gridCellsBuff!!.remaining() != accelSize) { + logger.warn("Skipping transmission this frame due to inconsistency in buffer size") + logger.warn("Size of color buffer: ${vdiColorBuffer!!.remaining()} and expected size $colorSize") + logger.warn("Size of color buffer: ${vdiDepthBuffer!!.remaining()} and expected size $depthSize") + logger.warn("Size of color buffer: ${gridCellsBuff!!.remaining()} and expected size $accelSize") + } else { + + if(Settings().get("Debug", false)) { + val floatBuffer = vdiColorBuffer!!.asFloatBuffer() + var cnt = 0 + while (floatBuffer.remaining() > 0) { + val t = floatBuffer.get() + if(t != 0f) { + cnt++ + } + } + if(cnt == 0) { + logger.warn("VDI color buffer only contains 0s.") + } + } + + if (compressedColor == null) { + compressedColor = + MemoryUtil.memAlloc(compressor.returnCompressBound(colorSize.toLong())) + } + + val compressedColorLength = + compressor.compress(compressedColor!!, vdiColorBuffer!!, 3) + compressedColor!!.limit(compressedColorLength.toInt()) + vdiData.bufferSizes.colorSize = compressedColorLength + + if (compressedDepth == null) { + compressedDepth = + MemoryUtil.memAlloc(compressor.returnCompressBound(depthSize.toLong())) + } + + val compressedDepthLength = + compressor.compress(compressedDepth!!, vdiDepthBuffer!!, 3) + compressedDepth!!.limit(compressedDepthLength.toInt()) + vdiData.bufferSizes.depthSize = compressedDepthLength + + val metadataOut = ByteArrayOutputStream() + VDIDataIO.write(vdiData, metadataOut) + + val metadataBytes = metadataOut.toByteArray() + logger.info("Size of VDI data is: ${metadataBytes.size}") + + val vdiDataSize = metadataBytes.size.toString().toByteArray(Charsets.US_ASCII) + + var messageLength = vdiDataSize.size + metadataBytes.size + compressedColor!!.remaining() + messageLength += compressedDepth!!.remaining() + messageLength += accelSize + + val message = ByteArray(messageLength) + vdiDataSize.copyInto(message) + + metadataBytes.copyInto(message, vdiDataSize.size) + + compressedColor!!.slice() + .get(message, vdiDataSize.size + metadataBytes.size, compressedColor!!.remaining()) + compressedDepth!!.slice().get( + message, + vdiDataSize.size + metadataBytes.size + compressedColor!!.remaining(), + compressedDepth!!.remaining() + ) + + vdiData.bufferSizes.accelGridSize = accelSize.toLong() + + gridCellsBuff!!.get( + message, vdiDataSize.size + metadataBytes.size + compressedColor!!.remaining() + + compressedDepth!!.remaining(), gridCellsBuff!!.remaining() + ) + gridCellsBuff!!.flip() + + compressedDepth!!.limit(compressedDepth!!.capacity()) + compressedColor!!.limit(compressedColor!!.capacity()) + + val sent = publisher.send(message) + if (!sent) { + logger.warn("There was a ZeroMQ error in queuing the VDI") + } else { + vdisStreamed += 1 + } + } + } + firstFrame = false + } + } + + private fun decompress( + payload: ByteArray, + compressedColor: ByteBuffer, + compressedDepth: ByteBuffer, + accelGridBuffer: ByteBuffer, + colorBuffer: ByteBuffer, + depthBuffer: ByteBuffer, + colorSize: Int, + depthSize: Int, + compressor: DataCompressor + ): VDIData { + + val metadataSize = payload.sliceArray(0 until 3).toString(Charsets.US_ASCII).toInt() //hardcoded 3 digit number + val metadata = ByteArrayInputStream(payload.sliceArray(3 until (metadataSize + 3))) + val vdiData = VDIDataIO.read(metadata) + logger.info("Index of received VDI: ${vdiData.metadata.index}") + + val compressedColorLength = vdiData.bufferSizes.colorSize + val compressedDepthLength = vdiData.bufferSizes.depthSize + + compressedColor.put(payload.sliceArray((metadataSize + 3) until (metadataSize + 3 + compressedColorLength.toInt()))) + compressedColor.flip() + compressedDepth.put(payload.sliceArray((metadataSize + 3) + compressedColorLength.toInt() until (metadataSize + 3) + compressedColorLength.toInt() + compressedDepthLength.toInt())) + compressedDepth.flip() + + accelGridBuffer.put(payload.sliceArray((metadataSize + 3) + compressedColorLength.toInt() + compressedDepthLength.toInt() until payload.size)) + accelGridBuffer.flip() + + val colorDone = AtomicInteger(0) + + thread { + compressedColor.limit(compressedColorLength.toInt()) + val decompressedColorLength = compressor.decompress(colorBuffer, compressedColor.slice()) + compressedColor.limit(compressedColor.capacity()) + if (decompressedColorLength.toInt() != colorSize) { + logger.warn("Error decompressing color message. Decompressed length: $decompressedColorLength and desired size: $colorSize") + } + colorDone.incrementAndGet() + } + + compressedDepth.limit(compressedDepthLength.toInt()) + val decompressedDepthLength = compressor.decompress(depthBuffer, compressedDepth.slice()) + compressedDepth.limit(compressedDepth.capacity()) + if (decompressedDepthLength.toInt() != depthSize) { + logger.warn("Error decompressing depth message. Decompressed length: $decompressedDepthLength and desired size: $depthSize") + } + + while (colorDone.get() == 0) { + Thread.sleep(20) + } + + colorBuffer.limit(colorSize) + + return vdiData + } + + /** + * Receives VDIs from a network stream and replaces them in the scene. + * + * The function runs blocking to receive and update successive VDIs transmitted across the network. + * + * @param[vdiNode] The [VDINode] that is part of the scene to be rendered + * @param[address] The network address (name/IP address and port number) from which to receive the VDIs + * @param[renderer] The renderer for this application + * @param[windowWidth] Window width of the application window. + * @param[windowHeight] Window height of the application window. + * @param[numSupersegments] The maximum number of supersegments in any list of the VDI, i.e., its resolution along z + */ + fun receiveAndUpdate( + vdiNode: VDINode, + address: String, + renderer: Renderer, + windowWidth: Int, + windowHeight: Int, + numSupersegments: Int + ) { + val subscriber: ZMQ.Socket = context.createSocket(SocketType.SUB) + subscriber.isConflate = true + try { + subscriber.connect(address) + } catch (e: ZMQException) { + logger.warn("ZMQ Binding failed.") + } + subscriber.subscribe(ZMQ.SUBSCRIPTION_ALL) + + //Attaching initial empty textures to the second buffer so that it is not null + vdiNode.attachEmptyTextures(VDINode.DoubleBuffer.Second) + + val compressionTool = DataCompressor.CompressionTool.LZ4 + val compressor = DataCompressor(compressionTool) + + //the expected sizes of each buffer + val colorSize = windowWidth * windowHeight * numSupersegments * 4 * 4 + val depthSize = windowWidth * windowHeight * numSupersegments * 2 * 4 + val accelSize = (windowWidth/8) * (windowHeight/8) * numSupersegments * 4 + + val decompressionBuffer = 1024 + + val colorBuffer = MemoryUtil.memCalloc(colorSize + decompressionBuffer) + val depthBuffer = MemoryUtil.memCalloc(depthSize + decompressionBuffer) + + val compressedColor: ByteBuffer = + MemoryUtil.memAlloc(compressor.returnCompressBound(colorSize.toLong())) + val compressedDepth: ByteBuffer = + MemoryUtil.memAlloc(compressor.returnCompressBound(depthSize.toLong())) + val accelGridBuffer = + MemoryUtil.memAlloc(accelSize) + + var vdiData: VDIData + + while(!renderer.shouldClose) { + val payload: ByteArray? + logger.info("Waiting for VDI") + + val receiveTime = measureNanoTime { + payload = subscriber.recv() + } + logger.info("Time taken for the receive: ${receiveTime/1e9}") + + if (payload != null) { + vdiData = decompress( + payload, + compressedColor, + compressedDepth, + accelGridBuffer, + colorBuffer, + depthBuffer, + colorSize, + depthSize, + compressor + ) + + colorBuffer.limit(colorBuffer.remaining() - decompressionBuffer) + depthBuffer.limit(depthBuffer.remaining() - decompressionBuffer) + + vdiNode.updateVDI(vdiData, colorBuffer.slice(), depthBuffer.slice(), accelGridBuffer) + + colorBuffer.limit(colorBuffer.capacity()) + depthBuffer.limit(depthBuffer.capacity()) + + firstVDIReceived = false + vdiNode.visible = true + } + else { + logger.info("Payload received but is null") + } + logger.info("Received and updated VDI data") + } + } +} diff --git a/src/main/kotlin/graphics/scenery/volumes/vdi/VDIVolumeManager.kt b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIVolumeManager.kt new file mode 100644 index 0000000000..d44c861205 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/volumes/vdi/VDIVolumeManager.kt @@ -0,0 +1,262 @@ +package graphics.scenery.volumes.vdi + +import bvv.core.shadergen.generate.SegmentTemplate +import bvv.core.shadergen.generate.SegmentType +import graphics.scenery.Hub +import graphics.scenery.RichNode +import graphics.scenery.Scene +import graphics.scenery.ShaderMaterial +import graphics.scenery.backends.Shaders +import graphics.scenery.compute.ComputeMetadata +import graphics.scenery.compute.InvocationType +import graphics.scenery.textures.Texture +import graphics.scenery.utils.Image +import graphics.scenery.utils.lazyLogger +import graphics.scenery.volumes.VolumeManager +import net.imglib2.type.numeric.integer.IntType +import net.imglib2.type.numeric.integer.UnsignedIntType +import net.imglib2.type.numeric.real.FloatType +import org.jetbrains.annotations.ApiStatus.Experimental +import org.joml.Vector3f +import org.joml.Vector3i +import org.lwjgl.system.MemoryUtil +import java.nio.ByteBuffer +import kotlin.math.ceil + +/** + * Class for creating and maintaining a [VolumeManager] for generating Volumetric Depth Images (VDIs). + * + * @param[hub] The hub to which the VolumeManager is to be attached + * @param[windowWidth] The rendering resolution along the x (horizontal) axis + * @param[windowHeight] The rendering resolution along the y (vertical) axis + * @param[maxSupersegments] The number of supersegments along each list (ray or pixel) in the generated VDI + * @param[scene] The scene to which the generated VolumeManager will belong + * + * @author Aryaman Gupta and Wissal Salhi + */ + +@Experimental +class VDIVolumeManager (var hub: Hub, val windowWidth: Int, val windowHeight: Int, val maxSupersegments: Int, val scene: Scene) +{ + private val logger by lazyLogger() + + var colorBuffer: ByteBuffer? = null + var depthBuffer: ByteBuffer? = null + var gridBuffer: ByteBuffer? = null + + var prefixBuffer: ByteBuffer? = null + var thresholdBuffer: ByteBuffer? = null + var numGeneratedBuffer: ByteBuffer? = null + + /** + * Creates a [VolumeManager] for generating VDIs + * + * @param[vdiFull] Boolean variable to indicate whether VDIs are to be generated in regular (i.e. full) resolution or compact. + * + * @return The generated [VolumeManager] + * + */ + fun createVDIVolumeManager(vdiFull: Boolean = true) : VolumeManager { + return if (vdiFull) + vdiFull(windowWidth, windowHeight, maxSupersegments, scene, hub) + else + vdiCompact(windowWidth, windowHeight, maxSupersegments, scene, hub) + } + + private fun instantiateVolumeManager(raycastShader: String, accumulateShader: String, hub: Hub): VolumeManager { + return VolumeManager( + hub, useCompute = true, + customSegments = hashMapOf( + SegmentType.FragmentShader to SegmentTemplate( + this::class.java, + raycastShader, + "intersectBoundingBox", "vis", "localNear", "localFar", "SampleVolume", "Convert", "Accumulate", + ), + SegmentType.Accumulator to SegmentTemplate( + accumulateShader, + "vis", "localNear", "localFar", "sampleVolume", "convert", "sceneGraphVisibility" + ), + ), + ) + } + + private fun vdiFull(windowWidth: Int, windowHeight: Int, maxSupersegments: Int, scene: Scene, hub: Hub): VolumeManager { + val raycastShader = "VDIGenerator.comp" + val accumulateShader = "AccumulateVDI.comp" + val volumeManager = instantiateVolumeManager(raycastShader, accumulateShader, hub) + + colorBuffer = MemoryUtil.memCalloc(windowHeight*windowWidth*4*maxSupersegments*4) + + depthBuffer = MemoryUtil.memCalloc(windowHeight*windowWidth*2*maxSupersegments*2 * 2) + + val numGridCells = Vector3f(windowWidth.toFloat() / 8f, windowHeight.toFloat() / 8f, maxSupersegments.toFloat()) + + gridBuffer = MemoryUtil.memCalloc(numGridCells.x.toInt() * numGridCells.y.toInt() * numGridCells.z.toInt() * 4) + + val vdiColor: Texture = Texture.fromImage( + Image(colorBuffer!!, maxSupersegments, windowHeight, windowWidth, FloatType()), usage = hashSetOf( Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + channels = 4, mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + volumeManager.customTextures.add(colorTextureName) + volumeManager.material().textures[colorTextureName] = vdiColor + + val vdiDepth: Texture = Texture.fromImage( + Image(depthBuffer!!, 2 * maxSupersegments, windowHeight, windowWidth, FloatType()), usage = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), + channels = 1, mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + volumeManager.customTextures.add(depthTextureName) + volumeManager.material().textures[depthTextureName] = vdiDepth + + val gridCells: Texture = Texture.fromImage( + Image(gridBuffer!!, numGridCells.x.toInt(), numGridCells.y.toInt(), numGridCells.z.toInt(), UnsignedIntType()), channels = 1, + usage = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + volumeManager.customTextures.add(accelerationTextureName) + volumeManager.material().textures[accelerationTextureName] = gridCells + + volumeManager.customUniforms.add("doGeneration") + volumeManager.shaderProperties["doGeneration"] = true + + val compute = RichNode() + compute.setMaterial(ShaderMaterial(Shaders.ShadersFromFiles(arrayOf("GridCellsToZero.comp"), this::class.java))) + + compute.metadata["ComputeMetadata"] = ComputeMetadata( + workSizes = Vector3i(numGridCells.x.toInt(), numGridCells.y.toInt(), 1), + invocationType = InvocationType.Permanent + ) + + compute.material().textures["GridCells"] = gridCells + + scene.addChild(compute) + + return volumeManager + } + + private fun vdiCompact(windowWidth: Int, windowHeight: Int, maxSupersegments: Int, scene: Scene, hub: Hub): VolumeManager { + val raycastShader = "AdaptiveVDIGenerator.comp" + val accumulateShader = "AccumulateVDI.comp" + + val volumeManager = instantiateVolumeManager(raycastShader, accumulateShader, hub) + + val totalMaxSupersegments = maxSupersegments * windowWidth * windowHeight + + colorBuffer = MemoryUtil.memCalloc(512 * 512 * ceil((totalMaxSupersegments / (512*512)).toDouble()).toInt() * 4 * 4) + depthBuffer = MemoryUtil.memCalloc(2 * 512 * 512 * ceil((totalMaxSupersegments / (512*512)).toDouble()).toInt() * 4) + + + val numGridCells = Vector3f(windowWidth.toFloat() / 8f, windowHeight.toFloat() / 8f, maxSupersegments.toFloat()) + gridBuffer = MemoryUtil.memCalloc(numGridCells.x.toInt() * numGridCells.y.toInt() * numGridCells.z.toInt() * 4) + + prefixBuffer = MemoryUtil.memCalloc(windowHeight * windowWidth * 4) + thresholdBuffer = MemoryUtil.memCalloc(windowHeight * windowWidth * 4) + numGeneratedBuffer = MemoryUtil.memCalloc(windowHeight * windowWidth * 4) + + val vdiColor: Texture = Texture.fromImage( + Image(colorBuffer!!, 512, 512, ceil((totalMaxSupersegments / (512*512)).toDouble()).toInt(), FloatType()), usage = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), channels = 4, mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + volumeManager.customTextures.add(colorTextureName) + volumeManager.material().textures[colorTextureName] = vdiColor + + val vdiDepth: Texture = Texture.fromImage( + Image(depthBuffer!!, 2 * 512, 512, ceil((totalMaxSupersegments / (512*512)).toDouble()).toInt(), FloatType()), usage = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture), channels = 1, mipmap = false, normalized = false, minFilter = Texture.FilteringMode.NearestNeighbour, maxFilter = Texture.FilteringMode.NearestNeighbour) + volumeManager.customTextures.add(depthTextureName) + volumeManager.material().textures[depthTextureName] = vdiDepth + + val gridCells: Texture = + Texture.fromImage(Image(gridBuffer!!, numGridCells.x.toInt(), numGridCells.y.toInt(), numGridCells.z.toInt(), FloatType()), channels = 1, + usage = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + volumeManager.customTextures.add(accelerationTextureName) + volumeManager.material().textures[accelerationTextureName] = gridCells + + volumeManager.customTextures.add("PrefixSums") + volumeManager.material().textures["PrefixSums"] = Texture( + Vector3i(windowHeight, windowWidth, 1), 1, contents = prefixBuffer, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture) + , type = IntType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + + volumeManager.customTextures.add("SupersegmentsGenerated") + volumeManager.material().textures["SupersegmentsGenerated"] = Texture( + Vector3i(windowHeight, windowWidth, 1), 1, contents = numGeneratedBuffer, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture) + , type = IntType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + + volumeManager.customTextures.add("Thresholds") + volumeManager.material().textures["Thresholds"] = Texture( + Vector3i(windowWidth, windowHeight, 1), 1, contents = thresholdBuffer, usageType = hashSetOf( + Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture) + , type = FloatType(), + mipmap = false, + minFilter = Texture.FilteringMode.NearestNeighbour, + maxFilter = Texture.FilteringMode.NearestNeighbour + ) + + volumeManager.customUniforms.add("doGeneration") + volumeManager.shaderProperties["doGeneration"] = false + + volumeManager.customUniforms.add("doThreshSearch") + volumeManager.shaderProperties["doThreshSearch"] = true + + volumeManager.customUniforms.add("windowWidth") + volumeManager.shaderProperties["windowWidth"] = windowWidth + + volumeManager.customUniforms.add("windowHeight") + volumeManager.shaderProperties["windowHeight"] = windowHeight + + volumeManager.customUniforms.add("maxSupersegments") + volumeManager.shaderProperties["maxSupersegments"] = maxSupersegments + + val compute = RichNode() + compute.setMaterial(ShaderMaterial(Shaders.ShadersFromFiles(arrayOf("GridCellsToZero.comp"), this::class.java))) + compute.metadata["ComputeMetadata"] = ComputeMetadata( + workSizes = Vector3i(numGridCells.x.toInt(), numGridCells.y.toInt(), 1), + invocationType = InvocationType.Permanent + ) + compute.material().textures["GridCells"] = gridCells + + scene.addChild(compute) + + return volumeManager + } + + /** + * Frees the memory allocated to the buffers used to generate VDIs. + */ + fun close() { + colorBuffer?.let { + MemoryUtil.memFree(it) + } + + depthBuffer?.let { + MemoryUtil.memFree(it) + } + + gridBuffer?.let { + MemoryUtil.memFree(it) + } + + prefixBuffer?.let { + MemoryUtil.memFree(it) + } + + thresholdBuffer?.let { + MemoryUtil.memFree(it) + } + + numGeneratedBuffer?.let { + MemoryUtil.memFree(it) + } + } + + companion object { + const val colorTextureName = "VDIColor" + const val depthTextureName = "VDIDepth" + const val accelerationTextureName = "AccelerationGrid" + + } +} diff --git a/src/main/resources/graphics/scenery/backends/DeferredShading.yml b/src/main/resources/graphics/scenery/backends/DeferredShading.yml index 19b0ed9512..2837624e5e 100644 --- a/src/main/resources/graphics/scenery/backends/DeferredShading.yml +++ b/src/main/resources/graphics/scenery/backends/DeferredShading.yml @@ -7,6 +7,7 @@ rendertargets: attachments: NormalsMaterial: RGBA_Float16 DiffuseAlbedo: RGBA_UInt8 + Emission: RGBA_Float16 ZBuffer: Depth32 ForwardBuffer: attachments: diff --git a/src/main/resources/graphics/scenery/backends/DeferredShadingGlitchy.yml b/src/main/resources/graphics/scenery/backends/DeferredShadingGlitchy.yml index 6b7d58e1f8..cbbe2a05ab 100644 --- a/src/main/resources/graphics/scenery/backends/DeferredShadingGlitchy.yml +++ b/src/main/resources/graphics/scenery/backends/DeferredShadingGlitchy.yml @@ -7,6 +7,7 @@ rendertargets: attachments: NormalsMaterial: RGBA_Float16 DiffuseAlbedo: RGBA_UInt8 + Emission: RGBA_Float16 ZBuffer: Depth32 ForwardBuffer: attachments: diff --git a/src/main/resources/graphics/scenery/backends/DeferredShadingStereo.yml b/src/main/resources/graphics/scenery/backends/DeferredShadingStereo.yml index 37b931ddac..3b4f58a15a 100644 --- a/src/main/resources/graphics/scenery/backends/DeferredShadingStereo.yml +++ b/src/main/resources/graphics/scenery/backends/DeferredShadingStereo.yml @@ -8,6 +8,7 @@ rendertargets: attachments: NormalsMaterial: RGBA_Float16 DiffuseAlbedo: RGBA_UInt8 + Emission: RGBA_Float16 ZBuffer: Depth32 ForwardBuffer: attachments: diff --git a/src/main/resources/graphics/scenery/backends/shaders/Atmosphere.frag b/src/main/resources/graphics/scenery/backends/shaders/Atmosphere.frag new file mode 100644 index 0000000000..c2e6d80d8d --- /dev/null +++ b/src/main/resources/graphics/scenery/backends/shaders/Atmosphere.frag @@ -0,0 +1,326 @@ +#version 450 +#extension GL_ARB_separate_shader_objects: enable + +layout(location = 0) in VertexData { + vec3 FragPosition; + vec3 Normal; + vec2 TexCoord; +} Vertex; + +layout(location = 0) out vec4 NormalsMaterial; +layout(location = 1) out vec4 DiffuseAlbedo; +layout(location = 2) out vec4 Emission; + +const float PI = 3.14159265358979323846264; +const int NUM_OBJECT_TEXTURES = 6; +#define iSteps 16 +#define jSteps 2 + +layout(set = 0, binding = 0) uniform VRParameters { + mat4 projectionMatrices[2]; + mat4 inverseProjectionMatrices[2]; + mat4 headShift; + float IPD; + int stereoEnabled; +} vrParameters; + +const int MAX_NUM_LIGHTS = 1024; + +struct Light { + float Linear; + float Quadratic; + float Intensity; + float Radius; + vec4 Position; + vec4 Color; +}; + +layout(set = 1, binding = 0) uniform LightParameters { + mat4 ViewMatrices[2]; + mat4 InverseViewMatrices[2]; + mat4 ProjectionMatrix; + mat4 InverseProjectionMatrix; + vec3 CamPosition; +}; + +struct MaterialInfo { + vec3 Ka; + vec3 Kd; + vec3 Ks; + float Roughness; + float Metallic; + float Opacity; + vec4 Emissive; +}; + +const int MATERIAL_HAS_DIFFUSE = 0x0001; +const int MATERIAL_HAS_AMBIENT = 0x0002; +const int MATERIAL_HAS_SPECULAR = 0x0004; +const int MATERIAL_HAS_NORMAL = 0x0008; +const int MATERIAL_HAS_ALPHAMASK = 0x0010; + +layout(set = 2, binding = 0) uniform Matrices { + mat4 ModelMatrix; + mat4 NormalMatrix; + int isBillboard; +} ubo; + +layout(set = 3, binding = 0) uniform MaterialProperties { + int materialType; + MaterialInfo Material; +}; + +layout(push_constant) uniform currentEye_t { + int eye; +} currentEye; + +/* + ObjectTextures[0] - ambient + ObjectTextures[1] - diffuse + ObjectTextures[2] - specular + ObjectTextures[3] - normal + ObjectTextures[4] - alpha + ObjectTextures[5] - displacement +*/ + +layout(set = 4, binding = 0) uniform sampler2D ObjectTextures[NUM_OBJECT_TEXTURES]; + +layout(set = 5, binding = 0) uniform ShaderProperties { + vec3 sunDir; +}; + +// courtesy of Christian Schueler - http://www.thetenthplanet.de/archives/1180 +mat3 TBN(vec3 N, vec3 position, vec2 uv) { + vec3 dp1 = dFdx(position); + vec3 dp2 = dFdy(position); + vec2 duv1 = dFdx(uv); + vec2 duv2 = dFdy(uv); + + vec3 dp2Perpendicular = cross(dp2, N); + vec3 dp1Perpendicular = cross(N, dp1); + + vec3 T = dp2Perpendicular * duv1.x + dp1Perpendicular * duv2.x; + vec3 B = dp2Perpendicular * duv1.y + dp1Perpendicular * duv2.y; + + float invmax = inversesqrt(max(dot(T, T), dot(B, B))); + + return transpose(mat3(T * invmax, B * invmax, N)); +} + +/* +Encodes a three component unit vector into a 2 component vector. The z component of the vector is stored, along with +the angle between the vector and the x axis. +*/ +vec2 EncodeSpherical(vec3 In) { + vec2 enc; + enc.x = atan(In.y, In.x) / PI; + enc.y = In.z; + enc = enc * 0.5f + 0.5f; + return enc; +} + +vec2 OctWrap( vec2 v ) +{ + vec2 ret; + ret.x = (1-abs(v.y)) * (v.x >= 0 ? 1.0 : -1.0); + ret.y = (1-abs(v.x)) * (v.y >= 0 ? 1.0 : -1.0); + return ret.xy; +} + +/* +Encodes a three component vector into a 2 component vector. First, a normal vector is projected onto one of the 8 planes +of an octahedron(|x| + |y| + |z| = 1). Then, the octahedron is orthogonally projected onto the xy plane to form a +square. The half of the octahedron where z is positive is projected directly by equating the z component to 0. The other +hemisphere is unfolded by splitting all edges adjacent to (0, 0, -1). The z component can be recovered while decoding by +using the property |x| + |y| + |z| = 1. +For more, refer to: http://www.vis.uni-stuttgart.de/~engelhts/paper/vmvOctaMaps.pdf. + */ +vec2 EncodeOctaH( vec3 n ) +{ + n /= ( abs( n.x ) + abs( n.y ) + abs( n.z )); + n.xy = n.z >= 0.0 ? n.xy : OctWrap( n.xy ); + n.xy = n.xy * 0.5 + 0.5; + return n.xy; +} + + +// Courtesy of Rye Terrell, https://github.com/wwwtyro/glsl-atmosphere + +//varying vec3 vPosition; + +vec2 rsi(vec3 r0, vec3 rd, float sr) { + // ray-sphere intersection that assumes + // the sphere is centered at the origin. + // No intersection when result.x > result.y + float a = dot(rd, rd); + float b = 2.0 * dot(rd, r0); + float c = dot(r0, r0) - (sr * sr); + float d = (b*b) - 4.0*a*c; + if (d < 0.0) return vec2(1e5,-1e5); + return vec2((-b - sqrt(d))/(2.0*a), (-b + sqrt(d))/(2.0*a)); +} + +vec3 atmosphere(vec3 r, vec3 r0, vec3 pSun, float iSun, float rPlanet, float rAtmos, vec3 kRlh, float kMie, float shRlh, float shMie, float g) { + // Normalize the sun and view directions. + pSun = normalize(pSun); + r = normalize(r); + + // Calculate the step size of the primary ray. + vec2 p = rsi(r0, r, rAtmos); + if (p.x > p.y) return vec3(0,0,0); + p.y = min(p.y, rsi(r0, r, rPlanet).x); + float iStepSize = (p.y - p.x) / float(iSteps); + + // Initialize the primary ray time. + float iTime = 0.0; + + // Initialize accumulators for Rayleigh and Mie scattering. + vec3 totalRlh = vec3(0,0,0); + vec3 totalMie = vec3(0,0,0); + + // Initialize optical depth accumulators for the primary ray. + float iOdRlh = 0.0; + float iOdMie = 0.0; + + // Calculate the Rayleigh and Mie phases. + float mu = dot(r, pSun); + float mumu = mu * mu; + float gg = g * g; + float pRlh = 3.0 / (16.0 * PI) * (1.0 + mumu); + float pMie = 3.0 / (8.0 * PI) * ((1.0 - gg) * (mumu + 1.0)) / (pow(1.0 + gg - 2.0 * mu * g, 1.5) * (2.0 + gg)); + + // Sample the primary ray. + for (int i = 0; i < iSteps; i++) { + + // Calculate the primary ray sample position. + vec3 iPos = r0 + r * (iTime + iStepSize * 0.5); + + // Calculate the height of the sample. + float iHeight = length(iPos) - rPlanet; + + // Calculate the optical depth of the Rayleigh and Mie scattering for this step. + float odStepRlh = exp(-iHeight / shRlh) * iStepSize; + float odStepMie = exp(-iHeight / shMie) * iStepSize; + + // Accumulate optical depth. + iOdRlh += odStepRlh; + iOdMie += odStepMie; + + // Calculate the step size of the secondary ray. + float jStepSize = rsi(iPos, pSun, rAtmos).y / float(jSteps); + + // Initialize the secondary ray time. + float jTime = 0.0; + + // Initialize optical depth accumulators for the secondary ray. + float jOdRlh = 0.0; + float jOdMie = 0.0; + + // Sample the secondary ray. + for (int j = 0; j < jSteps; j++) { + + // Calculate the secondary ray sample position. + vec3 jPos = iPos + pSun * (jTime + jStepSize * 0.5); + + // Calculate the height of the sample. + float jHeight = length(jPos) - rPlanet; + + // Accumulate the optical depth. + jOdRlh += exp(-jHeight / shRlh) * jStepSize; + jOdMie += exp(-jHeight / shMie) * jStepSize; + + // Increment the secondary ray time. + jTime += jStepSize; + } + + // Calculate attenuation. + vec3 attn = exp(-(kMie * (iOdMie + jOdMie) + kRlh * (iOdRlh + jOdRlh))); + + // Accumulate scattering. + totalRlh += odStepRlh * attn; + totalMie += odStepMie * attn; + + // Increment the primary ray time. + iTime += iStepSize; + + } + + // Calculate and return the final color. + return iSun * (pRlh * kRlh * totalRlh + pMie * kMie * totalMie); +} + + + +void main() { + DiffuseAlbedo.rgb = vec3(0.0f, 0.0f, 0.0f); + + DiffuseAlbedo.rgb = Material.Kd; + DiffuseAlbedo.a = 0.0f; + + NormalsMaterial.ba = vec2(Material.Roughness, Material.Metallic); + + //if((materialType & MATERIAL_HAS_AMBIENT) == MATERIAL_HAS_AMBIENT) { + // //DiffuseAlbedo.rgb = texture(ObjectTextures[0], VertexIn.TexCoord).rgb; + //} +// + //if((materialType & MATERIAL_HAS_DIFFUSE) == MATERIAL_HAS_DIFFUSE) { + // DiffuseAlbedo.rgb = texture(ObjectTextures[1], Vertex.TexCoord).rgb; + //} +// + //if((materialType & MATERIAL_HAS_SPECULAR) == MATERIAL_HAS_SPECULAR) { + // DiffuseAlbedo.a = texture(ObjectTextures[2], Vertex.TexCoord).r; + // NormalsMaterial.b = texture(ObjectTextures[2], Vertex.TexCoord).r; + //} +// + //if((materialType & MATERIAL_HAS_ALPHAMASK) == MATERIAL_HAS_ALPHAMASK) { + // if(texture(ObjectTextures[4], Vertex.TexCoord).r < 0.1f) { + // discard; + // } + //} + /* + Normals are encoded as Octahedron Normal Vectors, or Spherical Normal Vectors, which saves on storage as well as read/write processing of one + component. If using Spherical Encoding, do not forget to use spherical decode function in DeferredLighting shader. + */ + vec2 EncodedNormal = EncodeOctaH(Vertex.Normal); + // vec3 NormalizedNormal = normalize(VertexIn.Normal); + // vec2 EncodedNormal = EncodeSpherical(NormalizedNormal); + + + // if((materialType & MATERIAL_HAS_NORMAL) == MATERIAL_HAS_NORMAL) { + // vec3 normal = texture(ObjectTextures[3], Vertex.TexCoord).rgb*(255.0/127.0) - (128.0/127.0); + // normal = TBN(normalize(Vertex.Normal), CamPosition-Vertex.FragPosition, Vertex.TexCoord)*normal; + // + // EncodedNormal = EncodeOctaH(normal); + // } + + vec3 color = atmosphere( + normalize(Vertex.FragPosition), // normalized ray direction + vec3(0,6372e3,0), // ray origin + sunDir, // position of the sun + 22.0, // intensity of the sun + 6371e3, // radius of the planet in meters + 6471e3, // radius of the atmosphere in meters + vec3(5.5e-6, 13.0e-6, 22.4e-6), // Rayleigh scattering coefficient + 21e-6, // Mie scattering coefficient + 8e3, // Rayleigh scale height + 1.2e3, // Mie scale height + 0.758 // Mie preferred scattering direction + ); + + // Apply exposure. + color = 1.0 - exp(-1.0 * color); + + //float EmissionStrength = 1.0; + + //DiffuseAlbedo = vec4(0.5, 0.5, 1.0, 1.0); + DiffuseAlbedo = vec4(color, 1.0); + Emission.rgb = color; + Emission.a = Material.Emissive.a; + //emissive = EmissionStrength; //vec4(color, EmissionStrength); + NormalsMaterial.rg = EncodedNormal; +} + + + + +// https://github.com/wwwtyro/glsl-atmosphere/blob/master/example/example.frag diff --git a/src/main/resources/graphics/scenery/backends/shaders/Atmosphere.vert b/src/main/resources/graphics/scenery/backends/shaders/Atmosphere.vert new file mode 100644 index 0000000000..7f9eefa082 --- /dev/null +++ b/src/main/resources/graphics/scenery/backends/shaders/Atmosphere.vert @@ -0,0 +1,61 @@ +#version 450 core +#extension GL_ARB_separate_shader_objects: enable + +layout(location = 0) in vec3 vertexPosition; +layout(location = 1) in vec3 vertexNormal; +layout(location = 2) in vec2 vertexTexCoord; + +layout(location = 0) out VertexData { + vec3 FragPosition; + vec3 Normal; + vec2 TexCoord; +} Vertex; + + +layout(set = 0, binding = 0) uniform VRParameters { + mat4 projectionMatrices[2]; + mat4 inverseProjectionMatrices[2]; + mat4 headShift; + float IPD; + int stereoEnabled; +} vrParameters; + +layout(set = 1, binding = 0) uniform LightParameters { + mat4 ViewMatrices[2]; + mat4 InverseViewMatrices[2]; + mat4 ProjectionMatrix; + mat4 InverseProjectionMatrix; + vec3 CamPosition; +}; + +layout(set = 2, binding = 0) uniform Matrices { + mat4 ModelMatrix; + mat4 NormalMatrix; + int isBillboard; +} ubo; + +layout(push_constant) uniform currentEye_t { + int eye; +} currentEye; + +void main() +{ + mat4 mv; + mat4 nMVP; + mat4 projectionMatrix; + + mv = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye]); + projectionMatrix = (vrParameters.stereoEnabled ^ 1) * ProjectionMatrix + vrParameters.stereoEnabled * vrParameters.projectionMatrices[currentEye.eye]; + + nMVP = projectionMatrix*mv; + nMVP[3] = vec4(0.0f, 0.0f, 0.0f, 1.0f); + + Vertex.FragPosition = vec3(ubo.ModelMatrix * vec4(vertexPosition, 1.0)); + Vertex.Normal = mat3(ubo.NormalMatrix) * normalize(vertexNormal); + Vertex.TexCoord = vertexTexCoord; + + gl_PointSize = 1.0; + vec4 ndc = nMVP * vec4(vertexPosition, 1.0); + gl_Position = ndc.xyww; +} + diff --git a/src/main/resources/graphics/scenery/backends/shaders/DSSDO.frag b/src/main/resources/graphics/scenery/backends/shaders/DSSDO.frag index 60505a92cd..3598b8d65d 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/DSSDO.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/DSSDO.frag @@ -12,6 +12,7 @@ layout(set = 3, binding = 0) uniform sampler2D InputNormalsMaterial; layout(set = 3, binding = 1) uniform sampler2D InputDiffuseAlbedo; layout(set = 3, binding = 2) uniform sampler2D InputZBuffer; +layout(set = 3, binding = 3) uniform sampler2D InputEmission; layout(location = 0) out vec4 FragColor; layout(location = 0) in VertexData { diff --git a/src/main/resources/graphics/scenery/backends/shaders/DSSDOBlur.frag b/src/main/resources/graphics/scenery/backends/shaders/DSSDOBlur.frag index 5f4b528f2c..17c4eee1f6 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/DSSDOBlur.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/DSSDOBlur.frag @@ -12,6 +12,7 @@ layout(set = 1, binding = 0) uniform sampler2D InputNormalsMaterial; layout(set = 1, binding = 1) uniform sampler2D InputDiffuseAlbedo; layout(set = 1, binding = 2) uniform sampler2D InputZBuffer; +layout(set = 1, binding = 3) uniform sampler2D InputEmission; layout(set = 2, binding = 0) uniform sampler2D InputOcclusion; layout(location = 0) out vec4 FragColor; diff --git a/src/main/resources/graphics/scenery/backends/shaders/DefaultDeferred.frag b/src/main/resources/graphics/scenery/backends/shaders/DefaultDeferred.frag index 9b765ff005..e8a943d296 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/DefaultDeferred.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/DefaultDeferred.frag @@ -9,6 +9,7 @@ layout(location = 0) in VertexData { layout(location = 0) out vec4 NormalsMaterial; layout(location = 1) out vec4 DiffuseAlbedo; +layout(location = 3) out vec4 Emission; const float PI = 3.14159265358979323846264; const int NUM_OBJECT_TEXTURES = 6; @@ -47,6 +48,7 @@ struct MaterialInfo { float Roughness; float Metallic; float Opacity; + vec4 Emissive; }; const int MATERIAL_HAS_DIFFUSE = 0x0001; @@ -141,6 +143,8 @@ void main() { DiffuseAlbedo.rgb = Material.Kd; DiffuseAlbedo.a = 0.0f; + Emission = Material.Emissive; + NormalsMaterial.ba = vec2(Material.Roughness, Material.Metallic); if((materialType & MATERIAL_HAS_AMBIENT) == MATERIAL_HAS_AMBIENT) { diff --git a/src/main/resources/graphics/scenery/backends/shaders/DefaultForward.frag b/src/main/resources/graphics/scenery/backends/shaders/DefaultForward.frag index c2ee42183d..38448c9be3 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/DefaultForward.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/DefaultForward.frag @@ -36,7 +36,6 @@ layout(location = 0) in VertexData { vec3 FragPosition; } Vertex; - const int MATERIAL_HAS_DIFFUSE = 0x0001; const int MATERIAL_HAS_AMBIENT = 0x0002; const int MATERIAL_HAS_SPECULAR = 0x0004; diff --git a/src/main/resources/graphics/scenery/backends/shaders/DeferredLighting.frag b/src/main/resources/graphics/scenery/backends/shaders/DeferredLighting.frag index d8e582c513..6f64497693 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/DeferredLighting.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/DeferredLighting.frag @@ -13,7 +13,8 @@ layout(location = 0) out vec4 FragColor; layout(set = 3, binding = 0) uniform sampler2D InputNormalsMaterial; layout(set = 3, binding = 1) uniform sampler2D InputDiffuseAlbedo; -layout(set = 3, binding = 2) uniform sampler2D InputZBuffer; +layout(set = 3, binding = 3) uniform sampler2D InputZBuffer; +layout(set = 3, binding = 2) uniform sampler2D InputEmission; layout(set = 4, binding = 0) uniform sampler2D InputOcclusion; struct Light { @@ -485,6 +486,7 @@ void main() vec3 N = DecodeOctaH(texture(InputNormalsMaterial, textureCoord).rg); vec4 Albedo = texture(InputDiffuseAlbedo, textureCoord).rgba; + vec4 Emissive = texture(InputEmission, textureCoord).rgba; float Specular = texture(InputDiffuseAlbedo, textureCoord).a; vec2 MaterialParams = texture(InputNormalsMaterial, textureCoord).ba; @@ -615,8 +617,9 @@ void main() lighting = vec3(MaterialParams.rg, 0.0); } } else { - lighting = (diffuse + specular) * lightAttenuation; - } + lighting = (diffuse + specular) * lightAttenuation + Emissive.rgb * Emissive.a; + } + //} // check if occluded /* diff --git a/src/main/resources/graphics/scenery/backends/shaders/HBAO.frag b/src/main/resources/graphics/scenery/backends/shaders/HBAO.frag index 6ea2c371ca..f9194bc4c9 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/HBAO.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/HBAO.frag @@ -6,6 +6,7 @@ layout(set = 3, binding = 0) uniform sampler2D InputNormalsMaterial; layout(set = 3, binding = 1) uniform sampler2D InputDiffuseAlbedo; layout(set = 3, binding = 2) uniform sampler2D InputZBuffer; +layout(set = 3, binding = 3) uniform sampler2D InputEmission; layout(location = 0) out float FragColor; layout(location = 0) in VertexData { diff --git a/src/main/resources/graphics/scenery/backends/shaders/SSAO.frag b/src/main/resources/graphics/scenery/backends/shaders/SSAO.frag index e8674bcb69..356f78a56b 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/SSAO.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/SSAO.frag @@ -12,6 +12,7 @@ layout(set = 3, binding = 0) uniform sampler2D InputNormalsMaterial; layout(set = 3, binding = 1) uniform sampler2D InputDiffuseAlbedo; layout(set = 3, binding = 2) uniform sampler2D InputZBuffer; +layout(set = 3, binding = 3) uniform sampler2D InputEmission; layout(location = 0) out float FragColor; layout(location = 0) in VertexData { diff --git a/src/main/resources/graphics/scenery/ui/gear.png b/src/main/resources/graphics/scenery/ui/gear.png new file mode 100644 index 0000000000..f0d072670d Binary files /dev/null and b/src/main/resources/graphics/scenery/ui/gear.png differ diff --git a/src/main/resources/graphics/scenery/ui/gear.svg b/src/main/resources/graphics/scenery/ui/gear.svg new file mode 100644 index 0000000000..aaf9f70d6b --- /dev/null +++ b/src/main/resources/graphics/scenery/ui/gear.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag b/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag index 032864d171..5c6dba64aa 100644 --- a/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag +++ b/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag @@ -3,14 +3,16 @@ uniform int sceneGraphVisibility; vis = vis && bool(sceneGraphVisibility); -if (vis) +if (vis && step > localNear && step < localFar) { vec4 x = sampleVolume(wpos, volumeCache, cacheSize, blockSize, paddedBlockSize, cachePadOffset); float newAlpha = x.a; vec3 newColor = x.rgb; - v.rgb = v.rgb + (1.0f - v.a) * newColor * newAlpha; - v.a = v.a + (1.0f - v.a) * newAlpha; + float adjusted_alpha = adjustOpacity(newAlpha, (distance(wpos, wprev)/standardStepSize)); + + v.rgb = v.rgb + (1.0f - v.a) * newColor * adjusted_alpha; + v.a = v.a + (1.0f - v.a) * adjusted_alpha; if(v.a >= 1.0f) { break; diff --git a/src/main/resources/graphics/scenery/volumes/AccumulateSimpleVolume.frag b/src/main/resources/graphics/scenery/volumes/AccumulateSimpleVolume.frag index 9fa72da071..e622d60989 100644 --- a/src/main/resources/graphics/scenery/volumes/AccumulateSimpleVolume.frag +++ b/src/main/resources/graphics/scenery/volumes/AccumulateSimpleVolume.frag @@ -3,14 +3,17 @@ uniform int sceneGraphVisibility; vis = vis && bool(sceneGraphVisibility); -if (vis) +if (vis && step > localNear && step < localFar) { vec4 x = sampleVolume(wpos); + float newAlpha = x.a; vec3 newColor = x.rgb; - v.rgb = v.rgb + (1.0f - v.a) * newColor * newAlpha; - v.a = v.a + (1.0f - v.a) * newAlpha; + float adjusted_alpha = adjustOpacity(newAlpha, (distance(wpos, wprev)/standardStepSize)); + + v.rgb = v.rgb + (1.0f - v.a) * newColor * adjusted_alpha; + v.a = v.a + (1.0f - v.a) * adjusted_alpha; if(v.a >= 1.0f) { break; diff --git a/src/main/resources/graphics/scenery/volumes/BDVVolume.frag b/src/main/resources/graphics/scenery/volumes/BDVVolume.frag index bc257ef50c..d78f6a1a2e 100644 --- a/src/main/resources/graphics/scenery/volumes/BDVVolume.frag +++ b/src/main/resources/graphics/scenery/volumes/BDVVolume.frag @@ -46,6 +46,8 @@ layout(push_constant) uniform currentEye_t { } currentEye; #pragma scenery endverbatim +#extension GL_EXT_debug_printf : enable + // intersect ray with a box // http://www.siggraph.org/education/materials/HyperGraph/raytrace/rtinter3.htm void intersectBox( vec3 r_o, vec3 r_d, vec3 boxmin, vec3 boxmax, out float tnear, out float tfar ) @@ -64,6 +66,13 @@ void intersectBox( vec3 r_o, vec3 r_d, vec3 boxmin, vec3 boxmax, out float tnear tfar = min( min( tmax.x, tmax.y ), min( tmax.x, tmax.z ) ); } +float adjustOpacity(float a, float modifiedStepLength) { + return 1.0 - pow((1.0 - a), modifiedStepLength); +} + +uniform bool fixedStepSize; +uniform float stepsPerVoxel; + // --------------------- // $insert{Convert} // $insert{SampleVolume} @@ -83,7 +92,7 @@ void main() vec2 uv = Vertex.textureCoord * 2.0 - vec2(1.0); vec2 depthUV = (vrParameters.stereoEnabled ^ 1) * Vertex.textureCoord + vrParameters.stereoEnabled * vec2((Vertex.textureCoord.x/2.0 + currentEye.eye * 0.5), Vertex.textureCoord.y); depthUV = depthUV * 2.0 - vec2(1.0); - + // NDC of frag on near and far plane vec4 front = vec4( uv, -1, 1 ); vec4 back = vec4( uv, 1, 1 ); @@ -101,12 +110,16 @@ void main() float tnear = 1, tfar = 0, tmax = getMaxDepth( depthUV ); float n, f; - // $repeat:{vis,intersectBoundingBox| + // $repeat:{vis,localNear,localFar,intersectBoundingBox| bool vis = false; + float localNear = 0.0f; + float localFar = 0.0f; intersectBoundingBox( wfront, wback, n, f ); f = min( tmax, f ); if ( n < f ) { + localNear = n; + localFar = f; tnear = min( tnear, max( 0, n ) ); tfar = max( tfar, f ); vis = true; @@ -134,15 +147,28 @@ void main() ? int ( log( ( tfar * fwnw + nw ) / ( tnear * fwnw + nw ) ) / log ( 1 + fwnw ) ) : int ( trunc( ( tfar - tnear ) / nw + 1 ) ); + float stepWidth = nw; + + if(fixedStepSize) { + stepWidth = (2*nw) / stepsPerVoxel; + numSteps = int ( trunc( ( tfar - tnear ) / stepWidth + 1 ) ); + } + float step = tnear; + vec4 w_entry = mix(wfront, wback, step); + + float standardStepSize = distance(mix(wfront, wback, step + nw), w_entry); + + float step_prev = step - stepWidth; + vec4 wprev = mix(wfront, wback, step_prev); vec4 v = vec4( 0 ); - for ( int i = 0; i < numSteps; ++i, step += nw + step * fwnw ) + for ( int i = 0; i < numSteps; ++i) { vec4 wpos = mix( wfront, wback, step ); // $insert{Accumulate} /* - inserts something like the following (keys: vis,blockTexture,convert) + inserts something like the following (keys: vis,localNear,localFar,blockTexture,convert) if (vis) { @@ -150,10 +176,17 @@ void main() v = max(v, convert(x)); } */ + wprev = wpos; + + if(fixedStepSize) { + step += stepWidth; + } else { + step += nw + step * fwnw; + } } - v.xyz = pow(v.xyz, vec3(1/2.2)); FragColor = v; + if(v.w < 0.001f) { discard; } diff --git a/src/main/resources/graphics/scenery/volumes/ComputeHistogram.comp b/src/main/resources/graphics/scenery/volumes/ComputeHistogram.comp new file mode 100644 index 0000000000..7d68f834a4 --- /dev/null +++ b/src/main/resources/graphics/scenery/volumes/ComputeHistogram.comp @@ -0,0 +1,35 @@ +#version 450 + +layout (local_size_x = 16, local_size_y = 16) in; +layout (set = 0, binding = 0, r16f) uniform image3D Volume16Bit; +layout (set = 1, binding = 0, r8) uniform image3D Volume8Bit; +layout (set = 2, binding = 0, r32i) uniform iimage3D Histogram; + +layout(set = 3, binding = 0) uniform ShaderProperties { + bool volumeIs8Bit; + int numVoxels; + float maxDisplayVal; + float minDisplayVal; + int numBins; +}; + +void main() { + + float binSize = (maxDisplayVal - minDisplayVal) / numBins; + + for(int i = 0; i < numVoxels; i++) { + float val; + + if(volumeIs8Bit) { + val = imageLoad(Volume8Bit, ivec3(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y, i)).x * pow(2,8) ; + } else { + val = imageLoad(Volume16Bit, ivec3(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y, i)).x * pow(2,16); + } + + if(val >= minDisplayVal && val <= maxDisplayVal) { + float val2 = val - minDisplayVal; + int binID = int(floor(val2 / binSize)); + int ret = imageAtomicAdd(Histogram, ivec3(binID, 0, 0), 1); + } + } +} diff --git a/src/main/resources/graphics/scenery/volumes/VolumeRenderer.cl b/src/main/resources/graphics/scenery/volumes/VolumeRenderer.cl deleted file mode 100644 index ef4aba633e..0000000000 --- a/src/main/resources/graphics/scenery/volumes/VolumeRenderer.cl +++ /dev/null @@ -1,581 +0,0 @@ -/* -volume and iso surface rendering - - volume rendering adapted from the Nvidia sdk sample - http://developer.download.nvidia.com/compute/cuda/4_2/rel/sdk/website/OpenCL/html/samples.html - - - Author: Martin Weigert (mweigert@mpi-cbg.de) - Loic Royer (royer@mpi-cbg.de) -*/ - - -// Loop unrolling length: -#define LOOPUNROLL 16 - -// Typedefs: -//typedef unsigned int uint; -//typedef unsigned char uchar; - -// random number generator for dithering -inline -float random(uint x, uint y) -{ - uint a = 4421 +(1+x)*(1+y) +x +y; - - for(uint i=0; i < 10; i++) - { - a = ((uint)1664525 * a + (uint)1013904223) % (uint)79197919; - } - - float rnd = (a*1.0f)/(79197919.f); - - return rnd-0.5f; -} - - -// intersect ray with a box -// http://www.siggraph.org/education/materials/HyperGraph/raytrace/rtinter3.htm -inline -int intersectBox(float4 r_o, float4 r_d, float4 boxmin, float4 boxmax, float *tnear, float *tfar) -{ - // compute intersection of ray with all six bbox planes - float4 invR = (float4)(1.0f,1.0f,1.0f,1.0f) / r_d; - float4 tbot = invR * (boxmin - r_o); - float4 ttop = invR * (boxmax - r_o); - - // re-order intersections to find smallest and largest on each axis - float4 tmin = min(ttop, tbot); - float4 tmax = max(ttop, tbot); - - // find the largest tmin and the smallest tmax - float largest_tmin = max(max(tmin.x, tmin.y), max(tmin.x, tmin.z)); - float smallest_tmax = min(min(tmax.x, tmax.y), min(tmax.x, tmax.z)); - - *tnear = largest_tmin; - *tfar = smallest_tmax; - - return smallest_tmax > largest_tmin; -} - - -// convert float4 into uint: -inline -uint rgbaFloatToInt(float4 rgba) -{ - rgba = clamp(rgba,(float4)(0.f,0.f,0.f,0.f),(float4)(1.f,1.f,1.f,1.f)); - - return ((uint)(rgba.w*255)<<24) | ((uint)(rgba.z*255)<<16) | ((uint)(rgba.y*255)<<8) | (uint)(rgba.x*255); -} - -// convert float4 into uint and take the max with an existing RGBA value in uint form: -inline -uint rgbaFloatToIntAndMax(uint existing, float4 rgba) -{ - rgba = clamp(rgba,(float4)(0.f,0.f,0.f,0.f),(float4)(1.f,1.f,1.f,1.f)); - - const uint nr = (uint)(rgba.x*255); - const uint ng = (uint)(rgba.y*255); - const uint nb = (uint)(rgba.z*255); - const uint na = (uint)(rgba.w*255); - - const uint er = existing&0xFF; - const uint eg = (existing>>8)&0xFF; - const uint eb = (existing>>16)&0xFF; - const uint ea = (existing>>24)&0xFF; - - const uint r = max(nr,er); - const uint g = max(ng,eg); - const uint b = max(nb,eb); - const uint a = max(na,ea); - - return a<<24|b<<16|g<<8|r ; -} - - -// multiply matrix with vector -float4 mult(__constant float* M, float4 v){ - float4 res; - res.x = dot(v, (float4)(M[0],M[1],M[2],M[3])); - res.y = dot(v, (float4)(M[4],M[5],M[6],M[7])); - res.z = dot(v, (float4)(M[8],M[9],M[10],M[11])); - res.w = dot(v, (float4)(M[12],M[13],M[14],M[15])); - return res; -} - -__kernel void -dummy_render( - __global float4* d_output, - const uint imageW, - const uint imageH, - const float brightness, - const float trangemin, - const float trangemax, - const float gamma, - const float alpha_blending, - const int maxsteps, - const float dithering, - const float phase, - const int clear, - const float boxMin_x, - const float boxMax_x, - const float boxMin_y, - const float boxMax_y, - const float boxMin_z, - const float boxMax_z, - __constant float* invP, - __constant float* invM, - __read_only image3d_t volume) -{ - const uint x = get_global_id(0); - const uint y = get_global_id(1); - - d_output[x + y*imageW] = (float4)(0.0f, 0.0f, 1.0f, 1.0f); -} - -// Render function, -// performs max projection and then uses the transfert function to obtain a color per pixel: -__kernel void -maxproj_render( - __global float4 *d_output, - const uint imageW, - const uint imageH, - const float brightness, - const float trangemin, - const float trangemax, - const float gamma, - const float alpha_blending, - const int maxsteps, - const float dithering, - const float phase, - const int clear, - const float boxMin_x, - const float boxMax_x, - const float boxMin_y, - const float boxMax_y, - const float boxMin_z, - const float boxMax_z, - __read_only image2d_t transferColor4, - __constant float* invP, - __constant float* invM, - __read_only image3d_t volume) -{ - // samplers: - const sampler_t volumeSampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_LINEAR ; - const sampler_t transferSampler = CLK_NORMALIZED_COORDS_TRUE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_LINEAR ; - - // convert range bounds to linear map: - const float ta = 1.f/(trangemax-trangemin); - const float tb = trangemin/(trangemin-trangemax); - - // box bounds using the clipping box - const float4 boxMin = (float4)(boxMin_x,boxMin_y,boxMin_z,1.f); - const float4 boxMax = (float4)(boxMax_x,boxMax_y,boxMax_z,1.f); - - // thread int coordinates: - const uint x = get_global_id(0); - const uint y = get_global_id(1); - - if ((x >= imageW) || (y >= imageH)) return; - - // thread float coordinates: - const float u = (x / (float) imageW)*2.0f-1.0f; - const float v = (y / (float) imageH)*2.0f-1.0f; - - // front and back: - const float4 front = (float4)(u,v,-1.f,1.f); - const float4 back = (float4)(u,v,1.f,1.f); - - // calculate eye ray in world space - float4 orig0, orig; - float4 direc0, direc; - - orig0.x = dot(front, ((float4)(invP[0],invP[1],invP[2],invP[3]))); - orig0.y = dot(front, ((float4)(invP[4],invP[5],invP[6],invP[7]))); - orig0.z = dot(front, ((float4)(invP[8],invP[9],invP[10],invP[11]))); - orig0.w = dot(front, ((float4)(invP[12],invP[13],invP[14],invP[15]))); - - orig0 *= 1.f/orig0.w; - - orig.x = dot(orig0, ((float4)(invM[0],invM[1],invM[2],invM[3]))); - orig.y = dot(orig0, ((float4)(invM[4],invM[5],invM[6],invM[7]))); - orig.z = dot(orig0, ((float4)(invM[8],invM[9],invM[10],invM[11]))); - orig.w = dot(orig0, ((float4)(invM[12],invM[13],invM[14],invM[15]))); - - orig *= 1.f/orig.w; - - direc0.x = dot(back, ((float4)(invP[0],invP[1],invP[2],invP[3]))); - direc0.y = dot(back, ((float4)(invP[4],invP[5],invP[6],invP[7]))); - direc0.z = dot(back, ((float4)(invP[8],invP[9],invP[10],invP[11]))); - direc0.w = dot(back, ((float4)(invP[12],invP[13],invP[14],invP[15]))); - - direc0 *= 1.f/direc0.w; - - direc0 = normalize(direc0-orig0); - - direc.x = dot(direc0, ((float4)(invM[0],invM[1],invM[2],invM[3]))); - direc.y = dot(direc0, ((float4)(invM[4],invM[5],invM[6],invM[7]))); - direc.z = dot(direc0, ((float4)(invM[8],invM[9],invM[10],invM[11]))); - direc.w = 0.0f; - -// printf("%f %f %f %f\n", invP[0], invP[5], invP[10], invP[15]); -//printf("orig: %f %f %f\n", orig.x, orig.y, orig.z); -//printf("dir: %f %f %f\n", direc.x, direc.y, direc.z); - - // find intersection with box - float tnear, tfar; - const int hit = intersectBox(orig,direc, boxMin, boxMax, &tnear, &tfar); - if (!hit || tfar<=0) - { -// d_output[x+imageW*y] = (float4)(orig.x, orig.y, orig.z, 1.0f); - d_output[x+imageW*y] = (float4)(0.0f, 0.0f, 0.0f, 1.0f); - return; - } - -// d_output[x+imageW*y] = (float4)(tnear, tfar, 0.0f, 1.0f); -// return; - - // clamp to near plane: - if (tnear < 0.0f) tnear = 0.0f; - - // compute step size: - const float tstep = fabs(tnear-tfar)/((maxsteps/LOOPUNROLL)*LOOPUNROLL); - - // apply phase: - orig += phase*tstep*direc; - - // randomize origin point a bit: - const uint entropy = (uint)( 6779514*fast_length(orig) + 6257327*fast_length(direc) ); - orig += dithering*tstep*random(entropy+x,entropy+y)*direc; - - // precompute vectors: - const float4 vecstep = 0.5f*tstep*direc; - float4 pos = orig*0.5f+0.5f + tnear*0.5f*direc; - - // Loop unrolling setup: - const int unrolledmaxsteps = (maxsteps/LOOPUNROLL); - - // raycasting loop: - float maxp = 0.0f; - - float mappedVal; - - if (alpha_blending<=0.f) - { - // No alpha blending: - for(int i=0; i= imageW) || (y >= imageH)) return; - - // thread float coordinates: - const float u = (x / (float) imageW)*2.0f-1.0f; - const float v = (y / (float) imageH)*2.0f-1.0f; - - // front and back: - const float4 front = (float4)(u,v,-1.f,1.f); - const float4 back = (float4)(u,v,1.f,1.f); - - // calculate eye ray in world space - float4 orig0, orig; - float4 direc0, direc; - - orig0.x = dot(front, ((float4)(invP[0],invP[1],invP[2],invP[3]))); - orig0.y = dot(front, ((float4)(invP[4],invP[5],invP[6],invP[7]))); - orig0.z = dot(front, ((float4)(invP[8],invP[9],invP[10],invP[11]))); - orig0.w = dot(front, ((float4)(invP[12],invP[13],invP[14],invP[15]))); - - orig0 *= 1.f/orig0.w; - - orig.x = dot(orig0, ((float4)(invM[0],invM[1],invM[2],invM[3]))); - orig.y = dot(orig0, ((float4)(invM[4],invM[5],invM[6],invM[7]))); - orig.z = dot(orig0, ((float4)(invM[8],invM[9],invM[10],invM[11]))); - orig.w = dot(orig0, ((float4)(invM[12],invM[13],invM[14],invM[15]))); - - orig *= 1.f/orig.w; - - direc0.x = dot(back, ((float4)(invP[0],invP[1],invP[2],invP[3]))); - direc0.y = dot(back, ((float4)(invP[4],invP[5],invP[6],invP[7]))); - direc0.z = dot(back, ((float4)(invP[8],invP[9],invP[10],invP[11]))); - direc0.w = dot(back, ((float4)(invP[12],invP[13],invP[14],invP[15]))); - - direc0 *= 1.f/direc0.w; - - direc0 = normalize(direc0-orig0); - - direc.x = dot(direc0, ((float4)(invM[0],invM[1],invM[2],invM[3]))); - direc.y = dot(direc0, ((float4)(invM[4],invM[5],invM[6],invM[7]))); - direc.z = dot(direc0, ((float4)(invM[8],invM[9],invM[10],invM[11]))); - direc.w = 0.0f; - - - // find intersection with box - float tnear, tfar; - const int hit = intersectBox(orig,direc, boxMin, boxMax, &tnear, &tfar); - if (!hit || tfar<=0) - { - d_output[x+imageW*y] = 0.f; - return; - } - - // clamp to near plane: - if (tnear < 0.0f) tnear = 0.0f; - - // compute step size: - const float tstep = fabs(tnear-tfar)/((maxsteps/LOOPUNROLL)*LOOPUNROLL); - - // randomize origin point a bit: - const uint entropy = (uint)( 6779514*fast_length(orig) + 6257327*fast_length(direc) ); - orig += dithering*tstep*random(entropy+x,entropy+y)*direc; - - // precompute vectors: - const float4 vecstep = 0.5f*tstep*direc; - float4 pos = orig*0.5f+0.5f + tnear*0.5f*direc; - - // Loop unrolling setup: - const int unrolledmaxsteps = (maxsteps/LOOPUNROLL); - - // iso value: - float isoVal = mad(1.0f/ta,pow(0.5f,1.0f/gamma),-tb); - - // starting value: - float newVal = 0; //read_imagef(volume, volumeSampler, pos).x; - - // is iso surface value greater or lower: - bool isGreater = newVal>isoVal; - - bool hitIso = false; - - // first pass: - for(int i=0; iisoVal) != isGreater) - { - hitIso = true; - break; - } - - pos+=vecstep; - } - if (hitIso) break; - } - - - //early termination if iso surface not hit: - if (!hitIso) - { - d_output[x+imageW*y] = 0.f; - return; - } - - //second pass: - hitIso = false; - pos-=2*vecstep; - const float4 finevecstep = 3*vecstep/maxsteps; - for(int i=0; iisoVal) != isGreater) - { - hitIso = true; - break; - } - pos+=finevecstep; - } - if (hitIso) break; - } - /**/ - - - // find the real intersection point - float oldVal = read_imagef(volume, volumeSampler, pos-finevecstep).x; - float lam = (newVal - isoVal)/(newVal-oldVal); - pos += lam*vecstep; - - - // getting the normals and do some nice phong shading - - // build light vector: - float4 light = (float4)(-lightX,-lightY,-lightZ,0); - - float c_diffuse = 0.2; - float c_specular = 0.6; - - light = mult(invM,light); - light = fast_normalize(light); - - // compute lateral step for normal calculation: - const float latx = 1.0f/(get_image_width(volume)); - const float laty = 1.0f/(get_image_height(volume)); - const float latz = 1.0f/(get_image_depth(volume)); - - - // robust 2nd order normal estimation: - float4 normal; - normal.x = 2.f*read_imagef(volume,volumeSampler,pos+(float4)(latx,0,0,0)).x- - 2.f*read_imagef(volume,volumeSampler,pos+(float4)(-latx,0,0,0)).x+ - read_imagef(volume,volumeSampler,pos+(float4)(2.f*latx,0,0,0)).x- - read_imagef(volume,volumeSampler,pos+(float4)(-2.f*latx,0,0,0)).x; - - normal.y = 2.f*read_imagef(volume,volumeSampler,pos+(float4)(0,laty,0,0)).x- - 2.f*read_imagef(volume,volumeSampler,pos+(float4)(0,-laty,0,0)).x+ - read_imagef(volume,volumeSampler,pos+(float4)(0,2.f*laty,0,0)).x- - read_imagef(volume,volumeSampler,pos+(float4)(0,-2.f*laty,0,0)).x; - - normal.z = 2.f*read_imagef(volume,volumeSampler,pos+(float4)(0,0,latz,0)).x- - 2.f*read_imagef(volume,volumeSampler,pos+(float4)(0,0,-latz,0)).x+ - read_imagef(volume,volumeSampler,pos+(float4)(0,0,2.f*latz,0)).x- - read_imagef(volume,volumeSampler,pos+(float4)(0,0,-2.f*latz,0)).x; - - normal.w = 0; - - // flip normal if we are comming from values greater than isoVal... - normal = (1.f-2*isGreater)*fast_normalize(normal); - - // Blinn-Phong specular reflection: - //float diffuse = fmax(0.f,dot(light,normal)); - //float specular = native_powr(fmax(0.f,dot(fast_normalize(light+fast_normalize(direc)),fast_normalize(normal))),45); - - // Phong specular reflection: - float diffuse = fmax(0.f,dot(light,normal)); - float4 reflect = 2*dot(light,normal)*normal-light; - float specular = native_powr(fmax(0.f,dot(fast_normalize(reflect),fast_normalize(direc))),10); - - // Mapping to transfert function range and gamma correction: - const float mappedVal = gamma; - - // lookup in transfer function texture: - float4 color = read_imagef(transferColor4, transferSampler, (float2)(mappedVal,0.0f)); - - const float lighting = (c_diffuse*diffuse + (diffuse>0)*c_specular*specular); - - // Apply lighting: - const float3 lightcolor = (float3)(1.0f,1.0f,1.0f); - color.x = mad(lightcolor.x,lighting,color.x); - color.y = mad(lightcolor.y,lighting,color.y); - color.z = mad(lightcolor.z,lighting,color.z); - - color = brightness*color; - - //d_output[x + y*imageW] = rgbaFloatToIntAndMax(0,color); - d_output[x + y*imageW] = rgbaFloatToIntAndMax(clear*d_output[x + y*imageW],color); - -} - - - - -// clears a buffer -__kernel void -clearbuffer(__global uint *buffer, - uint imageW, - uint imageH) -{ - - // thread int coordinates: - const uint x = get_global_id(0); - const uint y = get_global_id(1); - - // clears buffer: - if ((x < imageW) || (y < imageH)) - buffer[x + y*imageW] = 0; -} diff --git a/src/main/resources/graphics/scenery/volumes/colormap-rb-darker.png b/src/main/resources/graphics/scenery/volumes/colormap-rb-darker.png new file mode 100644 index 0000000000..a13293e38a Binary files /dev/null and b/src/main/resources/graphics/scenery/volumes/colormap-rb-darker.png differ diff --git a/src/main/resources/graphics/scenery/volumes/colormap-red-blue.png b/src/main/resources/graphics/scenery/volumes/colormap-red-blue.png new file mode 100644 index 0000000000..7718fb4878 Binary files /dev/null and b/src/main/resources/graphics/scenery/volumes/colormap-red-blue.png differ diff --git a/src/main/resources/graphics/scenery/volumes/vdi/AccumulateVDI.comp b/src/main/resources/graphics/scenery/volumes/vdi/AccumulateVDI.comp new file mode 100644 index 0000000000..530e4575cf --- /dev/null +++ b/src/main/resources/graphics/scenery/volumes/vdi/AccumulateVDI.comp @@ -0,0 +1,101 @@ +if (vis && step > localNear && step < localFar) +{ + transparentSample = false; + vec4 x = sampleVolume(wpos); + + //TODO: the below check should be removed for efficiency + if(x.r > -0.5 || lastSample) { // we need to process this sample only if it is from a volume that actually exists at this sample point + + float newAlpha = x.a; + float w = adjustOpacity(newAlpha, length(wpos - wprev)); //TODO: jump length can be precalculated + + vec3 newColor = x.rgb; + + if(w <= minOpacity) { + transparentSample = true; + } + + if(supersegmentIsOpen) { + + vec4 jump_pos = mix(wfront, wback, stepWidth * steps_in_supseg); //TODO: jump length of single step can be precalculated and scaled to num steps in supseg + + float segLen = length(jump_pos - wfront); + supersegmentAdjusted.rgb = curV.rgb / curV.a; + supersegmentAdjusted.a = adjustOpacity(curV.a, 1.0/segLen); + + float diff = diffPremultiplied(supersegmentAdjusted, x); + + bool newSupSeg = false; + if(diff >= newSupSegThresh) { + newSupSeg = true; + } + + if((newSupSeg)) { //closing a supersegment + num_terminations++; + supersegmentIsOpen = false; + + supSegEndPoint = ndc_step; + steps_in_supseg = 0; + + if(thresh_found) { + if(!supsegs_written) { + writeSegAndGrid(supersegmentNum, supSegStartPoint, supSegEndPoint, wfront, wback, curV, stepWidth, + steps_trunc_trans, ipv, uv, grid_cell); + } + supersegmentNum++; + } + steps_trunc_trans = 0; + } + } + + if( (!supersegmentIsOpen) && (!transparentSample) ) { //opening a supersegment + + supersegmentIsOpen = true; + vec4 ndcStart = pv * wpos; + ndcStart *= 1. / ndcStart.w; + float start_step = ndcStart.z; + supSegStartPoint = start_step; + supseg_start_w = wpos; + curV = vec4( 0 ); //TODO: should this be x instead? + } + + if(supersegmentIsOpen) { + + curV.rgb = curV.rgb + (1 - curV.a) * newColor * w; + curV.a = curV.a + (1 - curV.a) * w; + + steps_in_supseg++; + + if(!transparentSample) { + steps_trunc_trans = steps_in_supseg; + w_prev_non_transp = wpos; + + float step_next = step + stepWidth; + vec4 wnext = mix(wfront, wback, step_next); + + ndcPos = pv * wnext; + ndcPos *= 1. / ndcPos.w; + ndc_step = ndcPos.z; + } + + } + + if(lastSample && supersegmentIsOpen) { //close the supersegment after the last sample is accumulated + + num_terminations++; + supersegmentIsOpen = false; + + supSegEndPoint = ndc_step; + steps_in_supseg = 0; + + if(thresh_found) { + if(!supsegs_written) { + writeSegAndGrid(supersegmentNum, supSegStartPoint, supSegEndPoint, wfront, wback, curV, stepWidth, + steps_trunc_trans, ipv, uv, grid_cell); + } + supersegmentNum++; + } + steps_trunc_trans = 0; + } + } +} diff --git a/src/main/resources/graphics/scenery/volumes/vdi/GridCellsToZero.comp b/src/main/resources/graphics/scenery/volumes/vdi/GridCellsToZero.comp new file mode 100644 index 0000000000..e3e91deffb --- /dev/null +++ b/src/main/resources/graphics/scenery/volumes/vdi/GridCellsToZero.comp @@ -0,0 +1,21 @@ +#version 450 + +layout (local_size_x = 16, local_size_y = 16) in; +layout (set = 0, binding = 0, r32ui) uniform uimage3D GridCells; + +/** + * This shader is used in the generation of the grid acceleration data structure used in the rendering of Volumetric + * Depth Images (VDIs). The cells of the grid data structure are incremented by means of an atomic increment during + * the generation of VDIs. This shader is used to reset the values to 0 before the generation of the next VDI begins. + */ + +void main() { + ivec3 imageCoords = imageSize(GridCells); + int num_cells_z = imageCoords.z; + + int cnt = 0; + + for(int i = 0; i < num_cells_z; i++) { + imageStore(GridCells, ivec3(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y, i), uvec4(0)); + } +} diff --git a/src/main/resources/graphics/scenery/volumes/vdi/RaycastVDI.comp b/src/main/resources/graphics/scenery/volumes/vdi/RaycastVDI.comp new file mode 100644 index 0000000000..8e90221417 --- /dev/null +++ b/src/main/resources/graphics/scenery/volumes/vdi/RaycastVDI.comp @@ -0,0 +1,1322 @@ +#version 450 +#define USE_PRINTF 1 +#define DOUBLE_BUFFER 1 + +#extension GL_EXT_debug_printf : disable +#if USE_PRINTF +#extension GL_EXT_debug_printf : enable +#endif + +layout(set = 0, binding = 0) uniform VRParameters { + mat4 projectionMatrices[2]; + mat4 inverseProjectionMatrices[2]; + mat4 headShift; + float IPD; + int stereoEnabled; +} vrParameters; + +const int MAX_NUM_LIGHTS = 1024; + +layout(set = 1, binding = 0) uniform LightParameters { + mat4 ViewMatrices[2]; + mat4 InverseViewMatrices[2]; + mat4 ProjectionMatrix; + mat4 InverseProjectionMatrix; + vec3 CamPosition; +}; + +layout(set = 5, binding = 0) uniform ShaderProperties { + mat4 ProjectionOriginal; + mat4 invProjectionOriginal; + mat4 ViewOriginal; + mat4 ViewOriginal2; + mat4 invViewOriginal; + mat4 invViewOriginal2; + bool useSecondBuffer; + mat4 invModel; + vec3 volumeDims; + float nw; + int vdiWidth; + int vdiHeight; + int totalGeneratedSupsegs; + float downImage; + bool do_subsample; + int max_samples; + float sampling_factor; + bool skip_empty; +}; + +layout(push_constant) uniform currentEye_t { + int eye; +} currentEye; + +layout (local_size_x = 16, local_size_y = 16) in; +layout(set = 2, binding = 0, rgba32f) uniform readonly image3D InputVDI; +layout(set = 3, binding = 0, rgba8) uniform image2D OutputViewport; +layout (set = 4, binding = 0, r32f) uniform readonly image3D DepthVDI; +layout (set = 6, binding = 0, r32ui) uniform readonly uimage3D AccelerationGrid; +#if DOUBLE_BUFFER +layout(set = 7, binding = 0, rgba32f) uniform readonly image3D InputVDI2; +layout (set = 8, binding = 0, r32f) uniform readonly image3D DepthVDI2; +layout (set = 9, binding = 0, r32ui) uniform readonly uimage3D AccelerationGrid2; +#endif +ivec2 debug_pixel = ivec2(256, 300); + +float adjustOpacity(float a, float modifiedStepLength) { + float b = pow((1.0 - a), modifiedStepLength); + return 1.0 - b; +} + +struct rayProperties { + ivec2 coords; + vec4 NDC_front; + vec4 NDC_back; + vec4 wfront; + vec4 wback; +}; + +mat4 pv_orig, ivp_orig; +int windowWidth, windowHeight; + +rayProperties originalRay; // the ray (i.e. list) from the original VDI that is currently being intersected +rayProperties newRay; // the ray from the new viewpoint that this kernel invocation is traversing + +vec4 front_orig; // the start point of newRay in the perspective space of the original viewpoint +vec4 back_orig; // the end point of newRay in the perspective space of the original viewpoint + +float orig_tnear; +float orig_tfar; + +// intersect ray with a box +// http://www.siggraph.org/education/materials/HyperGraph/raytrace/rtinter3.htm +void intersectBox( vec3 r_o, vec3 r_d, vec3 boxmin, vec3 boxmax, out float tnear, out float tfar){ + // compute intersection of ray with all six bbox planes + vec3 invR = 1 / r_d; // TODO: shouldn't r_d be a unit vector? And what if any component is 0? + vec3 tbot = invR * ( boxmin - r_o ); + vec3 ttop = invR * ( boxmax - r_o ); + + // re-order intersections to find smallest and largest on each axis + vec3 tmin = min(ttop, tbot); + vec3 tmax = max(ttop, tbot); + + // find the largest tmin and the smallest tmax + tnear = max( max( tmin.x, tmin.y ), max( tmin.x, tmin.z ) ); + tfar = min( min( tmax.x, tmax.y ), min( tmax.x, tmax.z ) ); +} + +void intersectBoundingBox_x_11_x_( vec4 wfront, vec4 wback, out float tnear, out float tfar ){ + vec4 mfront = invModel * wfront; + vec4 mback = invModel * wback; + intersectBox( mfront.xyz, (mback - mfront).xyz, vec3( 0, 0, 0 ), volumeDims, tnear, tfar); +} + +ivec2 prevList = ivec2(-1); +int prevIndex = -1; + +int maxSupersegments; + +ivec3 num_cells; + +float near_plane = 0.1; //TODO: get from the CPU +float far_plane = 20.0; + +float A = -1 * ((far_plane) / (far_plane - near_plane)); +float B = -1 * ((far_plane * near_plane) / (far_plane - near_plane)); + +float z_to_view(float z_n) { + float z_v = -1 * (B / (A + z_n)); + return z_v; +} + +float z_to_ndc(float z_v) { + float z_n = -1 * ((A * z_v + B)/z_v); + return z_n; +} + +int findZInterval_ndc(float z_val) { + float dist_from_front = z_val - (-1); + int interval_num = int(floor(dist_from_front / (2.0 / num_cells.z))); + return interval_num; +} + +int findZInterval_view(float z_ndc) { + float z_view = z_to_view(z_ndc); + float dist_from_front = abs(z_view - (-1 * near_plane)); + int interval_num = int(floor(dist_from_front / ((far_plane - near_plane) / num_cells.z))); + return interval_num; +} + +void findListNumber(vec4 wpos, out ivec2 listNum, out vec4 NDC_orig) { + //For this sample point, calculate ray number from original VDI + NDC_orig = pv_orig * wpos; + NDC_orig *= 1/NDC_orig.w; //TODO: check + if(NDC_orig.x < -1 || NDC_orig.x > 1 || NDC_orig.y < -1 || NDC_orig.y > 1 || NDC_orig.z < -1 || NDC_orig.z > 1) + { + listNum = ivec2(-1, -1); + NDC_orig = vec4(-1.); + return; // This sample point is not in the original viewport and therefore cannot be in the VDI + } + vec2 tex_orig = (NDC_orig.xy + 1) / 2.0; + listNum.x = int(round(tex_orig.x * vdiWidth)); //TODO: verify that this is correct. Maybe floor makes more sense? + listNum.y = int(round(tex_orig.y * vdiHeight)); //TODO: verify that this is correct. Maybe floor makes more sense? +} + +float getSupsegFront(ivec2 theList, int index) { + vec4 front; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + front = imageLoad(DepthVDI2, ivec3(index*2, theList.y, theList.x)); + } else { + front = imageLoad(DepthVDI, ivec3(index*2, theList.y, theList.x)); + } + #else + front = imageLoad(DepthVDI, ivec3(index*2, theList.y, theList.x)); + #endif + return front.x; +} + +float getSupsegBack(ivec2 theList, int index) { + vec4 back; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + back = imageLoad(DepthVDI2, ivec3(index*2 + 1, theList.y, theList.x)); + } else { + back = imageLoad(DepthVDI, ivec3(index*2 + 1, theList.y, theList.x)); + } + #else + back = imageLoad(DepthVDI, ivec3(index*2 + 1, theList.y, theList.x)); + #endif + return back.x; +} + +float getPrecedingDepth(ivec2 theList, int index) { + float ret = getSupsegBack(theList, index - 1); + + if(index >= 1) { + return ret; + } else { + return -50000.; // -inf + } +} + +void binSearch(ivec2 theList, float dist_to_orig, int start, int end, out bool supseg_found, out int index, out float depthStart, out float depthEnd) { + supseg_found = false; + int low = start; + int high = end; + + while(low <= high) { + index = (low + high)/2; + depthEnd = getSupsegBack(theList, index); + if(depthEnd == 0.0) { //TODO: improve empty detection mechanism + //this supersegment has not been filled + high = index - 1; + continue; + } + if(depthEnd < dist_to_orig) { //this supersegment is behind the sample point + low = index + 1; + } else { + float prevEnd = getPrecedingDepth(theList, index); + + if(prevEnd < dist_to_orig) { + // this is the supersegment + supseg_found = true; + break; + } else { + high = index - 1; + } + } + } +} + +void binSearch2(ivec2 theList, float dist_to_orig, int start, int end, out bool supseg_found, out int index, out float depthStart, out float depthEnd) { + supseg_found = false; + int low = start; + int high = end; + while(low <= high) { + index = (low + high)/2; + depthStart = getSupsegFront(theList, index); + if(depthStart == 0.0) { //TODO: improve empty detection mechanism + //this supersegment has not been filled + high = index - 1; + continue; + } + if(depthStart > dist_to_orig) { //this supersegment is behind the sample point + high = index - 1; + } else { + float nextStart = (index dist_to_orig) { + // this is the supersegment + supseg_found = true; + break; + } else { + low = index + 1; + } + } + } +} + +void nextSupersegmentInList(ivec2 theList, float exit_distance, bool dotPositive, out bool supseg_found, out int index, out float depthStart, out float depthEnd) { + supseg_found = false; + if(dotPositive) { + if(prevIndex == maxSupersegments - 1) { + return; + } + index = prevIndex + 1; + depthStart = getSupsegFront(theList, index); + if(depthStart < exit_distance && depthStart != 0) { + // we have found our next supseg + supseg_found = true; + depthEnd = getSupsegBack(theList, index); + } + } else { + if(prevIndex == 0) { + return; + } + index = prevIndex - 1; + depthEnd = getSupsegBack(theList, index); + if(depthEnd > exit_distance) { + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel){ + debugPrintfEXT("found next supersegment\n"); + } + #endif + supseg_found = true; + depthStart = getSupsegFront(theList, index); + } + } +} + +vec4 u, dir; + +//TODO: this function is probably not needed and the call in traverseSubsample should be replaced with opt_seeded version +void findFirstSupseg(ivec2 theList, float dist_to_orig, float exit_distance, out bool supseg_found, out int index, out float depthStart, out float depthEnd) { + supseg_found = false; + if(prevIndex == -1) { + binSearch(theList, dist_to_orig, 0, maxSupersegments - 1, supseg_found, index, depthStart, depthEnd); + } else { + + if(getSupsegBack(theList, prevIndex) == 0 || getSupsegBack(theList, prevIndex) >= dist_to_orig) { + //the supersegment we are searching for, if present, lies on the left of prevIndex + //we can check the end-point of the preceding supersegment + if((getPrecedingDepth(theList, prevIndex) < dist_to_orig) ) { //check if prevIndex is a candidate + if(getSupsegBack(theList, prevIndex) != 0) { // if prevIndex was filled, it is the supseg we are searching for + supseg_found = true; + index = prevIndex; + depthEnd = getSupsegBack(theList, index); + + } + else { + binSearch(theList, dist_to_orig, 0, prevIndex - 1, supseg_found, index, depthStart, depthEnd); + } + // why do this? + } + } else { + if(prevIndex < (maxSupersegments - 1)) { + if(getSupsegBack(theList, prevIndex + 1) >= dist_to_orig) { + supseg_found = true; + index = prevIndex + 1; + depthEnd = getSupsegBack(theList, index); + } else if(prevIndex < (maxSupersegments - 2)) { + binSearch(theList, dist_to_orig, prevIndex + 2, maxSupersegments - 1, supseg_found, index, depthStart, depthEnd); + } + } + } + + } + + depthStart = getSupsegFront(theList, index); + + if(depthStart > exit_distance) { + supseg_found = false; + } + +} +void findFirstSupseg_opt_seeded(ivec2 theList, float dist_to_orig, float exit_distance, bool dotPositive, out bool supseg_found, out int index, out float depthStart, out float depthEnd) { + supseg_found = false; + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("prevIdx: %d\n", prevIndex); + } + #endif + int bin_search_end = -1; + int bin_search_start = -1; + bool condition; + condition = prevIndex < 0; + if(condition){ + bin_search_end = maxSupersegments - 1; + bin_search_start = 0; + } else { + float sides[2]; + if(!dotPositive){ + sides[0]=getSupsegFront(theList, prevIndex); + sides[1]=getSupsegFront(theList, prevIndex+1); + sides[1]=((sides[1] == 0) || (prevIndex<=maxSupersegments-2))?5000:sides[1]; + sides[0] = (sides[0]==0)?-5000:sides[0]; + } else{ + sides[1] = getSupsegBack(theList, prevIndex); + sides[0] = getSupsegBack(theList, prevIndex - 1); + sides[0] = ((prevIndex - 1) >= 0)?sides[0]:-5000; + sides[1] = (sides[1] == 0)?5000:sides[1]; + } + int interval = int(sides[1]>=dist_to_orig)+int(sides[0]>=dist_to_orig); + if(interval==2){ + bin_search_start = 0; + bin_search_end=prevIndex-1; + }else if(interval ==0){ + bin_search_start = prevIndex+1; + bin_search_end=maxSupersegments-1; + }else{ + if(sides[1]<4995){ + supseg_found=true; + index=prevIndex; + depthStart=sides[1]; + }else{ + bin_search_start=prevIndex-int(dotPositive); + bin_search_end=maxSupersegments-1; + } + } + } + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("trying to binsearch theList: (%d, %d), dist_to_orig%f, bin_search_start%d, bin_search_end%d, maxSupersegments%d",theList, dist_to_orig, bin_search_start, bin_search_end, maxSupersegments); + } + #endif + if (bin_search_end != -1) + { + if(!dotPositive){ + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("binsearch2 theList: (%d, %d), dist_to_orig%f, bin_search_start%d, bin_search_end%d, maxSupersegments%d",theList, dist_to_orig, bin_search_start, bin_search_end, maxSupersegments); + } + #endif + binSearch2(theList, dist_to_orig, bin_search_start, bin_search_end, supseg_found, index, depthStart, depthEnd); + depthEnd = getSupsegBack(theList, index); + if(depthEndexit_distance) { supseg_found=false; } + return; + } + } + depthEnd = getSupsegBack(theList, index); + depthStart = getSupsegFront(theList, index); + if((dotPositive && depthStart > exit_distance) || (!dotPositive && depthEnd=0) ? true: false; + + float dist_to_orig = dist_to_orig_n; + float exit_distance = exit_distance_n; + + bool supseg_found = false; + + bool firstIteration = true; + + do { + float depthStart, depthEnd; + int index; + + if(firstIteration) { + findFirstSupseg_opt_seeded(theList, dist_to_orig, exit_distance,dotPositive, supseg_found, index, depthStart, depthEnd); + firstIteration = false; + } else { + nextSupersegmentInList(theList, exit_distance, dotPositive, supseg_found, index, depthStart, depthEnd); + } + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("For list: (%d, %d). Found supseg %d index %d. Start depth: %f and end depth: %f\n", theList, supseg_found, index, depthStart, depthEnd); + } + #endif + prevList = theList; + + prevIndex = index; //setting prevIndex even if no supersegment was found + + if(dotPositive) { + depthStart = max(dist_to_orig, depthStart); + depthEnd = min(exit_distance, depthEnd); + } else { + depthStart = min(dist_to_orig, depthStart); + depthEnd = max(exit_distance, depthEnd); + } + + if(supseg_found) { + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Found supseg: (%d, %d), %d\n", theList, index); + } + #endif + + float start_t = (depthStart - u.z) / dir.z; + float end_t = (depthEnd - u.z) / dir.z; + + if(dotPositive) + {start_point = u + dir * start_t; + end_point = u + dir * end_t;} + vec4 w_start_point = ivp_orig * start_point; + w_start_point *= 1./w_start_point.w; + + vec4 w_end_point = ivp_orig * end_point; + w_end_point *= 1./w_end_point.w; + + vec4 supseg_col; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + supseg_col = imageLoad(InputVDI2, ivec3(index, theList.y, theList.x)); + } else { + supseg_col = imageLoad(InputVDI, ivec3(index, theList.y, theList.x)); + } + #else + supseg_col = imageLoad(InputVDI, ivec3(index, theList.y, theList.x)); + #endif + + float length_in_supseg = distance(w_start_point, w_end_point); + + float alpha = adjustOpacity(supseg_col.a, length_in_supseg); + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Accumulated color: (%f, %f, %f, %f)", accumulatedColor.rgba); + } + #endif + + accumulatedColor.rgb = accumulatedColor.rgb + (1-accumulatedColor.a) * supseg_col.rgb * alpha; + accumulatedColor.a = accumulatedColor.a + (1-accumulatedColor.a) * alpha; + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("supsegCol: (%f, %f, %f, %f), alpha: %f", supseg_col.xyzw, alpha); + debugPrintfEXT("NDC start point: (%f, %f, %f, %f) and NDC end point: (%f, %f, %f, %f).", start_point.xyzw, end_point.xyzw); + debugPrintfEXT("World start point: (%f, %f, %f, %f) and end point: (%f, %f, %f, %f). Length: %f", w_start_point.xyzw, w_end_point.xyzw, length_in_supseg); + debugPrintfEXT("Accumulated color: (%f, %f, %f, %f)", accumulatedColor.rgba); + } + #endif + + if(accumulatedColor.a > 0.99) { + break; + } + } + } while(supseg_found); +} + +int stepX, stepY; +float tdeltaX, tdeltaY; //the delta intercepts for the ray on the supseg lists +vec4 finalColor = vec4(0); + +void traverse_subsample(vec4 world_entry, vec4 world_end, float total_distance, int num_samples) { + + float jump_size = total_distance / float(num_samples); + + float step = 0; + + float normalized_jump = jump_size / total_distance; + + for(int i = 1; i <= num_samples; i++) { + step = (normalized_jump + ((i - 1) * normalized_jump)); + vec4 wpos = mix(world_entry, world_end, step); + + vec4 npos = pv_orig * wpos; + npos *= 1. / npos.w; + vec2 tex = (npos.xy + 1) / 2.0; + + ivec2 list_coords; + list_coords.x = int(round(tex.x * vdiWidth)); + list_coords.y = int(round(tex.y * vdiHeight)); + + originalRay.coords = list_coords; + originalRay.NDC_front = vec4( npos.xy, -1, 1 ); + originalRay.NDC_back = vec4( npos.xy, 1, 1 ); + + originalRay.wfront = ivp_orig * originalRay.NDC_front; + originalRay.wfront *= 1.0 / originalRay.wfront.w; + + originalRay.wback = ivp_orig * originalRay.NDC_back; + originalRay.wback *= 1.0 / originalRay.wback.w; + + // -- bounding box intersection for all volumes ---------- + orig_tnear = 1, orig_tfar = 0; + float n, f; + + bool vis = false; + intersectBoundingBox_x_11_x_( originalRay.wfront, originalRay.wback, n, f ); + + if ( n < f ) + { + orig_tnear = min( orig_tnear, max( 0, n ) ); + orig_tfar = max( orig_tfar, f ); + vis = true; + } + + bool supseg_found; + int index; + float depth_start, depth_end; + + float dist_to_orig = npos.z; + + findFirstSupseg(list_coords, dist_to_orig, dist_to_orig, supseg_found, index, depth_start, depth_end); + + if(supseg_found) { + + vec4 supseg_start_w = ivp_orig * vec4(npos.xy, depth_start, 1); + supseg_start_w *= 1. / supseg_start_w.w; + + vec4 supseg_end_w = ivp_orig * vec4(npos.xy, depth_end, 1); + supseg_end_w *= 1. / supseg_end_w.w; + + + vec4 supseg_col; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + supseg_col = imageLoad(InputVDI2, ivec3(index, list_coords.y, list_coords.x)); + } else { + supseg_col = imageLoad(InputVDI, ivec3(index, list_coords.y, list_coords.x)); + } + #else + supseg_col = imageLoad(InputVDI, ivec3(index, list_coords.y, list_coords.x)); + #endif + + float alpha = adjustOpacity(supseg_col.a, jump_size); + finalColor.rgb = finalColor.rgb + (1-finalColor.a) * supseg_col.rgb * alpha; + finalColor.a = finalColor.a + (1-finalColor.a) * alpha; + } + } +} + +void traverse_cell(vec4 entry_point, ivec2 entry_list, vec3 cell_min, vec3 cell_max, out vec4 end_point, out ivec2 nextList) { + + const float max_x = max(cell_max.x, cell_min.x); + const float min_x = min(cell_max.x, cell_min.x); + const float max_y = max(cell_max.y, cell_min.y); + const float min_y = min(cell_max.y, cell_min.y); + + float nextX = entry_list.x + stepX * 0.5; + float nextY = entry_list.y + stepY * 0.5; + + float finalZ = dir.z > 0 ? cell_max.z : cell_min.z; //TODO: Should this be 0 instead of -1 + + float nextX_ndc = nextX / vdiWidth * 2.0 - 1.0; + float nextY_ndc = nextY / vdiHeight * 2.0 - 1.0; + + float tmaxX = dir.x != 0 ? abs((nextX_ndc - entry_point.x) / dir.x) : 10000000; + float tmaxY = dir.y != 0 ? abs((nextY_ndc - entry_point.y) / dir.y) : 10000000; + float tmaxZ = dir.z != 0 ? abs((finalZ - entry_point.z) / dir.z) : 10000000; + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("tmaxX: %f, tmaxY: %f, tmaxZ: %f\n", tmaxX, tmaxY, tmaxZ); + } + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Dirx: %f, diry: %f, dirz: %f\n", dir.x, dir.y, dir.z); + } + #endif + ivec2 currentList = entry_list; + nextList = currentList; + vec4 start_point = entry_point; + + bool finalIteration = false; + + while ( !finalIteration ) // loop over all the lists intersected by this ray + { + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Looping for list: (%d, %d)\n", currentList); + } + #endif + float incr = 0; + if(tmaxZ <= tmaxX && tmaxZ <= tmaxY) { + //z is lowest + /* this check only ensures that this list will be the last one intersected in this cell. For calculating the + end point in the list, and the next_list, we use the x or y intercept*/ + + finalIteration = true; + } + if(tmaxX < tmaxY) { + //x is the lowest + incr = tmaxX; + nextList.x += stepX; + tmaxX += tdeltaX; + } else if(tmaxY < tmaxX) { + //y is the lowest + incr = tmaxY; + nextList.y += stepY; + tmaxY += tdeltaY; + } else { + // x and y are equal + incr = tmaxX; + nextList.x += stepX; + nextList.y += stepY; + tmaxX += tdeltaX; + tmaxY += tdeltaY; + } + + end_point = entry_point + dir * incr; + bool terminate = false; + + if(stepX == 1) { + if(end_point.x > max_x) { //TODO: perhaps a delta needs to be added to the comparison + terminate = true; + } + } else { + if(end_point.x < min_x) { //TODO: this check can potentially be incorporated into the loop + terminate = true; + } + } + + if(stepY == 1) { + if(end_point.y > max_y) { + terminate = true; + } + } else { + if(end_point.y < min_y) { + terminate = true; + } + } + findSupsegsInList(currentList, start_point, end_point, finalColor); + + if(terminate) { + break; + } + if(finalColor.a > 0.99) { + break; + } + currentList = nextList; + start_point = end_point; + } +} + +float cellToViewToNDC(int cell_z) { + float dist_from_near = cell_z * ((far_plane - near_plane) / float(num_cells.z)); + float view_z = dist_from_near + near_plane; + view_z = -1 * view_z; + + float ndc_z = -1 * ((A * view_z + B) / view_z); + + return ndc_z; +} + +ivec3 findGridCell(vec4 position) { + ivec3 grid_coords; + + vec2 tex = (position.xy + vec2(1.)) / 2.; + + grid_coords.xy = ivec2(floor(tex * num_cells.xy)); + grid_coords.z = findZInterval_view(position.z); + + return grid_coords; +} + +void main() { + + ivec2 imageCoords = imageSize(OutputViewport); + + if(skip_empty) { + #if DOUBLE_BUFFER + if(useSecondBuffer){ + num_cells = imageSize(AccelerationGrid2).xyz; + } else { + num_cells = imageSize(AccelerationGrid).xyz; + } + #else + num_cells = imageSize(AccelerationGrid).xyz; + #endif + } else { + num_cells = ivec3(1); + } + + windowWidth = imageCoords.x; + windowHeight = imageCoords.y; + windowWidth = int(downImage * windowWidth); + windowHeight = int(downImage * windowHeight); + + ivec3 vdiCoords; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + vdiCoords = imageSize(InputVDI2); + } else { + vdiCoords = imageSize(InputVDI); + } + #else + vdiCoords = imageSize(InputVDI); + #endif + + maxSupersegments = vdiCoords.x; + + mat4 view; + + view = (vrParameters.stereoEnabled ^ 1) * ViewMatrices[0] + (vrParameters.stereoEnabled * ViewMatrices[currentEye.eye]); + + mat4 inverseProjection = (vrParameters.stereoEnabled ^ 1) * InverseProjectionMatrix + (vrParameters.stereoEnabled * vrParameters.inverseProjectionMatrices[currentEye.eye]); + mat4 inverseView = inverse(view); //TODO: Why not use InverseViewMatrices[] directly? + + highp mat4 ipv = inverseView * inverseProjection; + + mat4 matViewOrig; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + matViewOrig = ViewOriginal2; + } else { + matViewOrig = ViewOriginal; + } + #else + matViewOrig = ViewOriginal; + #endif + + mat4 matInvViewOrig; + #if DOUBLE_BUFFER + if(useSecondBuffer) { + matInvViewOrig = invViewOriginal2; + } else { + matInvViewOrig = invViewOriginal; + } + #else + matInvViewOrig = invViewOriginal; + #endif + + pv_orig = ProjectionOriginal * matViewOrig; + ivp_orig = matInvViewOrig * invProjectionOriginal; + + vec2 texcoord = gl_GlobalInvocationID.xy/vec2(windowWidth, windowHeight); //TODO: here the ww and wh need to remain as they are + if((texcoord.x > 1) || (texcoord.y > 1)) { + return; + } + vec2 uv = texcoord * 2.0 - vec2(1.0); + vec2 depthUV = (vrParameters.stereoEnabled ^ 1) * texcoord + vrParameters.stereoEnabled * vec2((texcoord.x/2.0 + currentEye.eye * 0.5), texcoord.y); + depthUV = depthUV * 2.0 - vec2(1.0); + newRay.coords.xy = ivec2(gl_GlobalInvocationID.xy); + // NDC of frag on near and far plane + newRay.NDC_front = vec4( uv, -1, 1 ); + newRay.NDC_back = vec4( uv, 1, 1 ); + // calculate eye ray in world space + newRay.wfront = ipv * newRay.NDC_front; + newRay.wfront *= 1.0 / newRay.wfront.w; + newRay.wback = ipv * newRay.NDC_back; + newRay.wback *= 1 / newRay.wback.w; + + front_orig = pv_orig * newRay.wfront; // start point of ray in NDC coordinates of original viewpoint + front_orig *= 1.0 / front_orig.w; + back_orig = pv_orig * newRay.wback; // end point of ray in NDC coordinates of original viewpoint + back_orig *= 1.0 / back_orig.w; + + // -- bounding box intersection for all volumes ---------- + float tnear = 1, tfar = 0, tmax = 0; //getMaxDepth( depthUV ); + float n, f; + + bool vis = false; + intersectBoundingBox_x_11_x_( newRay.wfront, newRay.wback, n, f ); + // f = min( tmax, f ); + if ( n < f ) + { + tnear = min( tnear, max( 0, n ) ); + tfar = max( tfar, f ); + vis = true; + } + if(!vis) { + imageStore(OutputViewport, ivec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y), vec4(0)); + return; + } + + float step = tnear; + + vec4 first_entry_pt, final_exit_pt; + + int cnt = 0; + ivec2 firstList; + + if(tnear > tfar) { + finalColor = vec4(0, 0, 0, 1); + return; + } else { + // calculate the value of an acceptable small step + //calculate min width of a supseg list in NDC + vec4 n_start = vec4(0, 0, 0, 1); + vec4 n_end = vec4((2. / max(vdiHeight, vdiWidth)), 0, 0, 1); + + vec4 w_start = ivp_orig * n_start; + w_start = w_start / w_start.w; + + vec4 w_end = ivp_orig * n_end; + w_end = w_end / w_end.w; + + vec4 view_orig = ViewOriginal * newRay.wfront; + + view_orig = view_orig / view_orig.w; + + // find the first supersegment + + vec4 wnear = mix(newRay.wfront, newRay.wback, tnear); + + //if this point is within original viewport, then this is the starting point for raycasting + //if not, we need to find the first point in this direction that is in the original viewport + vec4 ndc_near = pv_orig * wnear; + ndc_near *= 1. / ndc_near.w; + first_entry_pt = ndc_near; + + vec4 wfar = mix(newRay.wfront, newRay.wback, tfar); + + vec4 ndc_far = pv_orig * wfar; + ndc_far *= 1. / ndc_far.w; + final_exit_pt = ndc_far; + float eps = 0.0001; + vec3 rayDirection = (ndc_far - ndc_near).xyz; + + //TODO: is there a better way to do this? + if(rayDirection.x == 0) { + rayDirection.x = 0.000001; + } else if(rayDirection.y == 0) { + rayDirection.y = 0.000001; + } else if (rayDirection.z == 0) { + rayDirection.z = 0.000001; + } + + rayDirection = normalize(rayDirection); + + dir.xyz = rayDirection; + dir.w = 0; + + + if(ndc_near.x < -1 || ndc_near.x > 1 || ndc_near.y < -1 || ndc_near.y > 1 || ndc_near.z < -1 || ndc_near.z > 1) { + // the point is not in the original viewport + float d1, d2; + + // intersect the NDC frustum of the original viewpoint + intersectBox( ndc_near.xyz, rayDirection, vec3(-1.), vec3(1.), d1, d2); + + if(d1 > d2) { + //an error has occurred. This ray either doesn't pass through the original viewport or + // passes right at the edge, and floating point error is making the alg. believe it misses + + imageStore(OutputViewport, ivec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y), vec4(0)); + return; + } + + vec4 f_pt = vec4(ndc_near.xyz + rayDirection * d1, 1); + vec4 l_pt = vec4(ndc_near.xyz + rayDirection * d2, 1); + + vec4 f_pt_w = ivp_orig * f_pt; + f_pt_w *= 1. / f_pt_w.w; + vec4 l_pt_w = ivp_orig * l_pt; + l_pt_w *= 1. / l_pt_w.w; + + if(distance(f_pt_w, newRay.wfront) > distance(l_pt_w, newRay.wfront)) { + step = distance(l_pt_w, newRay.wfront) / distance(newRay.wfront, newRay.wback); + vec2 tex_orig = (l_pt.xy + 1) / 2.0; + + firstList.x = int(round(tex_orig.x * vdiWidth)); + firstList.y = int(round(tex_orig.y * vdiHeight)); + + first_entry_pt = l_pt; + final_exit_pt = f_pt; + } else { + step = distance(f_pt_w, newRay.wfront) / distance(newRay.wfront, newRay.wback); + vec2 tex_orig = (f_pt.xy + 1) / 2.0; + + firstList.x = int(round(tex_orig.x * vdiWidth)); + firstList.y = int(round(tex_orig.y * vdiHeight)); + + first_entry_pt = f_pt; + final_exit_pt = l_pt; + } + + } else { + vec4 ndc; + findListNumber(wnear, firstList, ndc); + step = tnear; + } + + front_orig = first_entry_pt; + back_orig = final_exit_pt; + + } + + ivec2 currentList = firstList; + u = first_entry_pt; + + stepX = dir.x > 0 ? 1 : -1; + stepY = dir.y > 0 ? 1 : -1; + int stepZ = dir.z > 0 ? 1 : -1; + + tdeltaX = abs((2.0 / vdiWidth) / dir.x); + tdeltaY = abs((2.0 / vdiHeight) / dir.y); + + const int MAX_X = 1; + const int MIN_X = -1; + + const int MAX_Y = 1; + const int MIN_Y = -1; + + const int MAX_Z = 1; + const int MIN_Z = -1; + + float cell_tdeltaX = abs((2. / num_cells.x) / dir.x); + float cell_tdeltaY = abs((2. / num_cells.y) / dir.y); + float cell_tdeltaZ = 0; + + vec4 end_point = vec4(0); + vec4 start_point = vec4(0); + vec4 end_traversal = vec4(0); + ivec2 start_list, next_list; + ivec3 grid_cell, next_cell; + float cell_tmaxX, cell_tmaxY, cell_tmaxZ; + bool finalIteration = false; + + int num_traversed = 0; + + int num_octree_traversals = 1; + if(do_subsample) { + num_octree_traversals = 2; + } + float length_in_non_empty = 0; + + int calculatedMaxSteps = 0; + + for(int trav = 1; trav <= num_octree_traversals; trav++) { + + start_point = first_entry_pt; + + vec2 tex_orig = (start_point.xy + 1.0) / 2.0; + + start_list.x = int(round(tex_orig.x * vdiWidth)); + start_list.y = int(round(tex_orig.y * vdiHeight)); + + next_list = start_list; + + // ivec3 grid_cell = findGridCell(start_list, start_point.z); + grid_cell = findGridCell(start_point); + next_cell = grid_cell; + next_cell.x += (stepX == 1 ? 1 : 0); + next_cell.y += (stepY == 1 ? 1 : 0); + next_cell.z += (stepZ == 1 ? 1 : 0); + + vec3 next_cell_ndc; + next_cell_ndc.xy = ((next_cell.xy / vec2(num_cells.xy)) * 2.0) - vec2(1.0); + next_cell_ndc.z = cellToViewToNDC(next_cell.z); + + // cell_tmaxX = dir.x != 0 ? abs((next_cell_ndc.x - start_point.x) / dir.x) : 10000000; + // cell_tmaxY = dir.y != 0 ? abs((next_cell_ndc.y - start_point.y) / dir.y) : 10000000; + // cell_tmaxZ = dir.z != 0 ? abs((next_cell_ndc.z - start_point.z) / dir.z) : 10000000; + + cell_tmaxX = abs((next_cell_ndc.x - start_point.x) / dir.x); + cell_tmaxY = abs((next_cell_ndc.y - start_point.y) / dir.y); + cell_tmaxZ = abs((next_cell_ndc.z - start_point.z) / dir.z); + + float t = 0; + end_traversal = vec4(0); + + finalIteration = false; + bool cell_traversed = false; + + while(!finalIteration) { //looping over the cells of the lowest level of the octree + + if(cell_traversed) { + //if the new cell is to start at the end point of the previous traversal, then we need to check whether the + //previous traversal took us to a different grid cell along z + + // ivec3 grid_at_end = findGridCell(start_list, start_point.z); + if(start_point.x < MIN_X || start_point.x > MAX_X || start_point.y < MIN_Y || start_point.y > MAX_Y || start_point.z < MIN_Z || start_point.z > MAX_Z) { + break; + } + + ivec3 grid_at_end = findGridCell(start_point); + + if(grid_at_end.x != grid_cell.x) { + t = cell_tmaxX; + cell_tmaxX += cell_tdeltaX; + } + + if(grid_at_end.y != grid_cell.y) { + t = cell_tmaxY; + cell_tmaxY += cell_tdeltaY; + } + + if(grid_at_end.z != grid_cell.z) { + t = cell_tmaxZ; + float current_z_ndc = cellToViewToNDC(grid_at_end.z); + int next_z = grid_at_end.z + (stepZ == 1 ? 1 : -1); + float next_z_ndc = cellToViewToNDC(next_z); + cell_tdeltaZ = dir.z != 0 ? abs((next_z_ndc - current_z_ndc) / dir.z) : 10000000; + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("current z: %d, next z: %d, delta obtained: %f, prev tmaxZ was: %f", grid_cell.z, grid_at_end.z, cell_tdeltaZ, cell_tmaxZ); + } + #endif + cell_tmaxZ += cell_tdeltaZ; + } + + + grid_cell = grid_at_end; + + + } + + cell_traversed = false; + + vec3 cell_min; + cell_min.xy = vec2(grid_cell.xy / vec2(num_cells.xy)); + cell_min.xy = (cell_min.xy * 2.) - 1.; + cell_min.z = cellToViewToNDC(grid_cell.z); + vec3 cell_max; + cell_max.xy = vec2((grid_cell.xy+vec2(1.)) / vec2(num_cells.xy)); + cell_max.xy = (cell_max.xy * 2.) - 1.; + cell_max.z = cellToViewToNDC(grid_cell.z + 1); + + uint grid_val; + + if(skip_empty) { + #if DOUBLE_BUFFER + if(useSecondBuffer) { + grid_val = imageLoad(AccelerationGrid2, grid_cell).r; + } else { + grid_val = imageLoad(AccelerationGrid, grid_cell).r; + } + #else + grid_val = imageLoad(AccelerationGrid, grid_cell).r; + #endif + } else { + grid_val = 1; + } + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Checking the cell: (%d, %d, %d). value: %d", grid_cell.xyz, grid_val); + } + #endif + + if(grid_val > 0 && !do_subsample) { + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("traversing the cell: (%d, %d, %d), cell min: (%f, %f, %f), cell max: (%f, %f, %f), start point: (%f, %f, %f). value: %d\n", grid_cell.xyz, cell_min.xyz, cell_max.xyz, start_point.xyz, grid_val); + } + #endif + + traverse_cell(start_point, start_list, cell_min, cell_max, end_traversal, next_list); + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("end point: (%f, %f, %f)", end_point.xyz); + } + #endif + cell_traversed = true; + } + + //TODO: the next section can probably be skipped if the cell has been traversed + + // changed this from always true to !cell_traversed + if(!cell_traversed) { + + #if USE_PRINTF + if (gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("cell_tmaxX: %f, cell_tmaxY: %f, cell_tmaxZ: %f\n", cell_tmaxX, cell_tmaxY, cell_tmaxZ); + } + #endif + + + if(cell_tmaxX < cell_tmaxY) { + if(cell_tmaxX < cell_tmaxZ) { + // x is the lowest + t = cell_tmaxX; + grid_cell.x += stepX; + if(grid_cell.x >= num_cells.x || grid_cell.x < 0) { // TODO: only one comparison is required depending on stepX + finalIteration = true; + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("This is final iteration because new x val: %d:", grid_cell.x); + } + #endif + } + cell_tmaxX += cell_tdeltaX; + } else { + // z is lowest or x = z + t = cell_tmaxZ; + grid_cell.z += stepZ; + if(grid_cell.z >= num_cells.z || grid_cell.z < 0) { + finalIteration = true; + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("This is final iteration because new z val: %d:", grid_cell.z); + } + #endif + } + + int next_z = grid_cell.z + (stepZ == 1 ? 1 : -1); + float current_z_ndc = cellToViewToNDC(grid_cell.z); + float next_z_ndc = cellToViewToNDC(next_z); + cell_tdeltaZ = dir.z != 0 ? abs((next_z_ndc - current_z_ndc) / dir.z) : 10000000; + + cell_tmaxZ += cell_tdeltaZ; + } + } else { + if(cell_tmaxY < cell_tmaxZ) { + // y is the lowest + t = cell_tmaxY; + grid_cell.y += stepY; + if(grid_cell.y >= num_cells.y || grid_cell.y < 0) { + finalIteration = true; + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("This is final iteration because new y val: %d:", grid_cell.y); + } + #endif + } + cell_tmaxY += cell_tdeltaY; + } else { + // z is the lowest or y = z + t = cell_tmaxZ; + grid_cell.z += stepZ; + if(grid_cell.z >= num_cells.z || grid_cell.z < 0) { + finalIteration = true; + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("This is final iteration because new z val: %d:", grid_cell.z); + } + #endif + } + + int next_z = grid_cell.z + (stepZ == 1 ? 1 : -1); + float current_z_ndc = cellToViewToNDC(grid_cell.z); + float next_z_ndc = cellToViewToNDC(next_z); + cell_tdeltaZ = dir.z != 0 ? abs((next_z_ndc - current_z_ndc) / dir.z) : 10000000; + + cell_tmaxZ += cell_tdeltaZ; + } + } + + end_point = first_entry_pt + t * dir; + tex_orig = (end_point.xy + 1.0)/2.0; + + next_list.x = int(round(tex_orig.x * vdiWidth)); + next_list.y = int(round(tex_orig.y * vdiHeight)); + } else { + end_point = end_traversal; + //we have next_list from the traversal method + } + + uint supsegs_in_cell; + if(skip_empty) { + supsegs_in_cell = imageLoad(AccelerationGrid, grid_cell).r; + } else { + supsegs_in_cell = 0; + } + + if(supsegs_in_cell > 0 && do_subsample) { + vec4 cell_start_w = ivp_orig * start_point; + cell_start_w *= 1. / cell_start_w.w; + + vec4 cell_end_w = ivp_orig * end_point; + cell_end_w *= 1. / cell_end_w.w; + float intersection_length = distance(cell_start_w, cell_end_w); + if(trav == 1) { + length_in_non_empty += supsegs_in_cell * intersection_length; + } else { + float fract = (supsegs_in_cell * intersection_length) / length_in_non_empty; + // int samples_in_cell = int(fract * max_samples); //TODO: should it be round()? + int samples_in_cell = int(fract * calculatedMaxSteps); //TODO: should it be round()? + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Cell: (%d, %d, %d), intersection length: %f and supseg count: %d. Steps: %d will be taken. Max samples: %d and overall length_in_non_empty: %f calculated max steps: %d", grid_cell.xyz, intersection_length, + int(supsegs_in_cell), samples_in_cell, max_samples, length_in_non_empty, calculatedMaxSteps); + } + #endif + + traverse_subsample(cell_start_w, cell_end_w, intersection_length, samples_in_cell); + num_traversed++; + } + } + + + //TODO: add check for outside the volume even though still inside the VDI + + if(finalColor.a > 0.99) { + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Breaking because alpha is: %f", finalColor.a); + } + #endif + break; + } + + start_point = end_point; + start_list = next_list; + } + calculatedMaxSteps = int(length_in_non_empty * sampling_factor); + } + + finalColor.xyz = pow(finalColor.xyz, vec3(1/2.2)); + + imageStore(OutputViewport, ivec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y), finalColor); + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Final!!! composited color is: (%f, %f, %f, %f)", finalColor.rgba); + } + #endif +} diff --git a/src/main/resources/graphics/scenery/volumes/vdi/VDIGenerator.comp b/src/main/resources/graphics/scenery/volumes/vdi/VDIGenerator.comp new file mode 100644 index 0000000000..2177ebbff8 --- /dev/null +++ b/src/main/resources/graphics/scenery/volumes/vdi/VDIGenerator.comp @@ -0,0 +1,431 @@ +uniform vec2 viewportSize; +uniform vec2 dsp; +uniform float fwnw; +uniform float nw; + +// -- comes from CacheSpec ----- +uniform vec3 blockSize; +uniform vec3 paddedBlockSize; +uniform vec3 cachePadOffset; + +// -- comes from TextureCache -- +uniform vec3 cacheSize; // TODO: get from texture!? +uniform mat4 transform; + +uniform bool doGeneration; + +#pragma scenery verbatim +layout(set = 0, binding = 0) uniform VRParameters { + mat4 projectionMatrices[2]; + mat4 inverseProjectionMatrices[2]; + mat4 headShift; + float IPD; + int stereoEnabled; +} vrParameters; + +const int MAX_NUM_LIGHTS = 1024; + +layout(set = 1, binding = 0) uniform LightParameters { + mat4 ViewMatrices[2]; + mat4 InverseViewMatrices[2]; + mat4 ProjectionMatrix; + mat4 InverseProjectionMatrix; + vec3 CamPosition; +}; + +layout(push_constant) uniform currentEye_t { + int eye; +} currentEye; + +#define USE_PRINTF 0 + +#if USE_PRINTF +#extension GL_EXT_debug_printf : enable +#endif + +layout(local_size_x = 16, local_size_y = 16) in; +layout(set = 2, binding = 0) uniform sampler3D volumeCache; +layout(set = 3, binding = 0, rgba32f) uniform image3D VDIColor; +layout(set = 4, binding = 0, r32f) uniform image3D VDIDepth; +layout (set = 8, binding = 0, r32ui) uniform uimage3D AccelerationGrid; + +#pragma scenery endverbatim + +ivec2 debug_pixel = ivec2(360, 360); + +// intersect ray with a box +// http://www.siggraph.org/education/materials/HyperGraph/raytrace/rtinter3.htm +void intersectBox( vec3 r_o, vec3 r_d, vec3 boxmin, vec3 boxmax, out float tnear, out float tfar ) +{ + // compute intersection of ray with all six bbox planes + vec3 invR = 1 / r_d; + vec3 tbot = invR * ( boxmin - r_o ); + vec3 ttop = invR * ( boxmax - r_o ); + + // re-order intersections to find smallest and largest on each axis + vec3 tmin = min(ttop, tbot); + vec3 tmax = max(ttop, tbot); + + // find the largest tmin and the smallest tmax + tnear = max( max( tmin.x, tmin.y ), max( tmin.x, tmin.z ) ); + tfar = min( min( tmax.x, tmax.y ), min( tmax.x, tmax.z ) ); +} + +float adjustOpacity(float a, float modifiedStepLength) { + return 1.0 - pow((1.0 - a), modifiedStepLength); +} + +float diffPremultiplied(vec4 a, vec4 b) { + a.rgb = a.rgb * a.a; + b.rgb = b.rgb * b.a; + + return length(a.rgb-b.rgb); +} + +vec4 diffComponentWise(vec4 a, vec4 b) { + a.rgb = a.rgb * a.a; + b.rgb = b.rgb * b.a; + + vec4 diff = abs(a - b); + + diff /= a; + + return diff; +} + +float diffRelative(vec4 supseg, vec4 new_sample) { + supseg.rgb = supseg.rgb * supseg.a; + new_sample.rgb = new_sample.rgb * new_sample.a; + + return (length(supseg.rgb-new_sample.rgb) / length(supseg.rgb)); +} + +const vec4 bitEnc = vec4(1.,255.,65025.,16581375.); +vec4 EncodeFloatRGBA (float v) { + vec4 enc = bitEnc * v; + enc = fract(enc); + enc -= enc.yzww * vec2(1./255., 0.).xxxy; + return enc; +} + +vec4 encode(float x, float y){ + vec4 rgba; + + x += 128.; + y += 128.; + + int ix = int( x * 256. ); // convert to int to split accurately + int v1x = ix / 256; // hi + int v1y = ix - v1x * 256; // lo + + rgba.r = float( v1x + 1 ) / 255.; // normalize + rgba.g = float( v1y + 1 ) / 255.; + + int iy = int( y * 256. ); + int v2x = iy / 256; // hi + int v2y = iy - v2x * 256; // lo + + rgba.b = float( v2x + 1 ) / 255.; + rgba.a = float( v2y + 1 ) / 255.; + + return rgba - 1./256.; +} + +void writeSupersegment(int index, float start, float end, vec4 color) { + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Writing supseg: %d. Start: %f, end: %f, color: (%f, %f, %f, %f)", index, start, end, color.rgba); + } + if(isnan(color.r) || isnan(color.a) || isnan(start) || isnan(end) || isnan(index)) { + debugPrintfEXT("Error! Wrong supersegment written by: (%d, %d)", gl_GlobalInvocationID.xy); + } + #endif + + imageStore(VDIDepth, ivec3(2 * index, gl_GlobalInvocationID.y, gl_GlobalInvocationID.x), vec4(start, 0, 0, 0)); + imageStore(VDIDepth, ivec3(2 * index + 1, gl_GlobalInvocationID.y, gl_GlobalInvocationID.x), vec4(end, 0, 0, 0)); + imageStore(VDIColor, ivec3(index, gl_GlobalInvocationID.y, gl_GlobalInvocationID.x), color); +} + +void update_cell_count(ivec3 cell) { + uint ret = imageAtomicAdd(AccelerationGrid, cell, 1); +} + +ivec3 num_cells; +float near_plane = 0.1; //TODO: generalize +float far_plane = 20.0; + +int findZInterval_view(float z_view) { + + float dist_from_front = abs(z_view - (-1 * near_plane)); + float interval_size = ((far_plane - near_plane) / num_cells.z); + int interval_num = int(floor(dist_from_front / interval_size)); + + return interval_num; +} + +bool thresh_found = false; + +void writeSegAndGrid(int supersegmentNum, float supSegStartPoint, float supSegEndPoint, vec4 wfront, vec4 wback, + vec4 accumulatedColor, float stepWidth, int steps, mat4 ipv, vec2 uv, ivec3 grid_cell) { + + vec4 jump_pos = mix(wfront, wback, stepWidth * steps); + + float segLen = length(jump_pos - wfront); + + vec4 supersegmentColor; + + supersegmentColor.rgb = accumulatedColor.rgb / accumulatedColor.a; + supersegmentColor.a = adjustOpacity(accumulatedColor.a, 1.0/segLen); + + writeSupersegment(supersegmentNum, supSegStartPoint, supSegEndPoint, supersegmentColor); + + vec4 start_w = ipv * vec4(uv, supSegStartPoint, 1); + start_w *= 1. / start_w.w; + + vec4 end_w = ipv * vec4(uv, supSegEndPoint, 1); + end_w *= 1. / end_w.w; + + vec4 start_v = ViewMatrices[0] * start_w; + vec4 end_v = ViewMatrices[0] * end_w; + + int start_cell = findZInterval_view(start_v.z); + int end_cell = findZInterval_view(end_v.z); + + for(int j = start_cell; j <= end_cell; j++) { + grid_cell.z = j; + update_cell_count(grid_cell); + } +} + +// --------------------- +// $insert{Convert} +// $insert{SampleVolume} +// --------------------- + +void main() +{ + if(!doGeneration) { + return; + } + float stepWidth = nw; + + ivec3 imageCoords = imageSize(VDIColor); + int windowWidth = imageCoords.b; + int windowHeight = imageCoords.g; + + num_cells = imageSize(AccelerationGrid).xyz; + + + ivec3 grid_cell = ivec3(0); + grid_cell.x = int(floor((float(gl_GlobalInvocationID.x) / windowWidth) * num_cells.x)); + grid_cell.y = int(floor((float(gl_GlobalInvocationID.y) / windowHeight) * num_cells.y)); + + mat4 ipv = InverseViewMatrices[0] * InverseProjectionMatrix; + mat4 pv = ProjectionMatrix * ViewMatrices[0]; + + // frag coord in NDC + // TODO: Re-introduce dithering + // vec2 fragCoord = (vrParameters.stereoEnabled ^ 1) * gl_FragCoord.xy + vrParameters.stereoEnabled * vec2((gl_FragCoord.x/2.0 + currentEye.eye * gl_FragCoord.x/2.0), gl_FragCoord.y); + // vec2 viewportSizeActual = (vrParameters.stereoEnabled ^ 1) * viewportSize + vrParameters.stereoEnabled * vec2(viewportSize.x/2.0, viewportSize.y); + // vec2 uv = 2 * ( gl_FragCoord.xy + dsp ) / viewportSizeActual - 1; + // float newSupSegThresh = 0.00014; + float newSupSegThresh = 0.04555; + + vec2 texcoord = gl_GlobalInvocationID.xy/vec2(imageCoords.b, imageCoords.g); + vec2 uv = texcoord * 2.0 - vec2(1.0); + vec2 depthUV = (vrParameters.stereoEnabled ^ 1) * texcoord + vrParameters.stereoEnabled * vec2((texcoord.x/2.0 + currentEye.eye * 0.5), texcoord.y); + depthUV = depthUV * 2.0 - vec2(1.0); + + vec4 FragColor = vec4(0.0); + + // NDC of frag on near and far plane + vec4 front = vec4( uv, -1, 1 ); + vec4 back = vec4( uv, 1, 1 ); + + // calculate eye ray in world space + vec4 wfront = ipv * front; + wfront *= 1 / wfront.w; + vec4 wback = ipv * back; + wback *= 1 / wback.w; + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Num grid cells: (%d, %d, %d).", num_cells.xyz); + } + #endif + + // -- bounding box intersection for all volumes ---------- + float tnear = 1, tfar = 0, tmax = getMaxDepth( depthUV ); + float n, f; + + // $repeat:{vis,localNear,localFar,intersectBoundingBox| + bool vis = false; + float localNear = 0.0f; + float localFar = 0.0f; + intersectBoundingBox( wfront, wback, n, f ); + f = min( tmax, f ); + if ( n < f ) + { + localNear = n; + localFar = f; + tnear = min( tnear, max( 0, n ) ); + tfar = max( tfar, f ); + vis = true; + } + // }$ + + // ------------------------------------------------------- + + int maxSupersegments = imageCoords.r; + + float minOpacity = 0.0; //If alpha is less than this, the sample is considered transparent and not included in generated supersegments + // float minOpacity = 0.00196078431; //If alpha is less than this, the sample is considered transparent and not included in generated supersegments + /* Exlanation of minOpacity value: the smallest number that can be stored in 8 but opacity channel is 1/255 = 0.00392156862. Any value less than half of this will be rounded down to 0 and + therefore not impact the rendering. 0.00392156862/2 = 0.00196078431*/ + + int supersegmentNum = 0; + + if ( tnear < tfar ) + { + vec4 fb = wback - wfront; + int numSteps = int ( trunc( ( tfar - tnear ) / stepWidth ) ); + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("tnear: %f, tfar: %f, nw: %f. numSteps: %d.", tnear, tfar, nw, numSteps); + } + #endif + + float low_thresh = 0.0; + float high_thresh = 1.732; //sq. root of 3 + + bool final_generation_step = false; + bool supsegs_written = false; + bool error_computed = false; + + int desired_supsegs = maxSupersegments; + int delta = int(floor(0.15 * maxSupersegments)); // up to delta supsegs less than max is acceptable + + int iter = 0; + float mid_thresh = 0.0001; //start off with a very low thresh to eliminate those rays that contain primarily homogenous regions already + bool first_iteration = true; + + while(!thresh_found || !supsegs_written) { + iter++; + newSupSegThresh = mid_thresh; + + int num_terminations = 0; + + bool supersegmentIsOpen = false; + float supSegStartPoint = 0.0; + float supSegEndPoint = 0.0; + bool lastSample = false; + bool transparentSample = false; + bool lastSupersegment = false; + vec4 supersegmentAdjusted = vec4(0); + + float step = tnear; + float step_prev = step - stepWidth; + vec4 wprev = mix(wfront, wback, step_prev); + vec4 w_prev_non_transp = vec4(0); + + vec4 ndcPos; + float ndc_step; + + int steps_in_supseg = 0; + int steps_trunc_trans = 0; + + vec4 curV = vec4( 0 ); + vec4 supseg_start_w = vec4(0); + + for ( int i = 0; i < numSteps; ++i, step += stepWidth ) + { + if(i==(numSteps-1)) { + lastSample = true; + } + + if(supersegmentNum == (maxSupersegments - 1)) { + lastSupersegment = true; + } + + vec4 wpos = mix( wfront, wback, step ); + + vec4 ro_world, rd_world; + + + // $insert{Accumulate} + /* + inserts something like the following (keys: vis,localNear,localFar,blockTexture,convert) + + if (vis) + { + float x = blockTexture(wpos, volumeCache, cacheSize, blockSize, paddedBlockSize, cachePadOffset); + v = max(v, convert(x)); + } + */ + + wprev = wpos; + } + + if(supsegs_written) { + error_computed = true; + } + + if(thresh_found) { + supsegs_written = true; + } + + #if USE_PRINTF //TODO: check if the iterations are actually taking place and benchmark against fixed threshold + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Iteration: %d of searching for thresh. Low: %f, high: %f, mid: %f. Num terminations: %d", iter, low_thresh, high_thresh, mid_thresh, num_terminations); + debugPrintfEXT("Desired supsegs: %d and delta: %d", desired_supsegs, delta); + } + #endif + + if(!supsegs_written) { + + if(abs(high_thresh - low_thresh) < 0.000001) { + thresh_found = true; + mid_thresh = ((num_terminations == 0) ? low_thresh : high_thresh); // we want to err on the higher side, so that we generate < max no of supsegs, unless we are ending up generating 0 supsegs + continue; + } else if(num_terminations > desired_supsegs) { + low_thresh = mid_thresh; + } else if(num_terminations < (desired_supsegs - delta)) { + high_thresh = mid_thresh; + } else { + thresh_found = true; + continue; + } + + if(first_iteration) { + first_iteration = false; + if(num_terminations < desired_supsegs) { + thresh_found = true; + continue; + } + } + + mid_thresh = (low_thresh + high_thresh) / 2.0; + } + } + + + #if USE_PRINTF + if(gl_GlobalInvocationID.xy == debug_pixel) { + debugPrintfEXT("Final composited color is: (%f, %f, %f, %f)", v.rgba); + debugPrintfEXT("Total supsegs generated: %d", supersegmentNum); + } + #endif + if(supersegmentNum < maxSupersegments) { + for(int i = supersegmentNum; i < maxSupersegments; i++) { + writeSupersegment(i, 0, 0, vec4(0)); + } + } + } else { + if(supersegmentNum < maxSupersegments) { + for(int i = supersegmentNum; i < maxSupersegments; i++) { + writeSupersegment(i, 0, 0, vec4(0)); + } + } + } +} diff --git a/src/test/java/graphics/scenery/tests/applications/vdi/ClientApplication.kt b/src/test/java/graphics/scenery/tests/applications/vdi/ClientApplication.kt new file mode 100644 index 0000000000..1141296463 --- /dev/null +++ b/src/test/java/graphics/scenery/tests/applications/vdi/ClientApplication.kt @@ -0,0 +1,227 @@ +package graphics.scenery.tests.applications.vdi + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.textures.Texture +import graphics.scenery.ui.SwingBridgeFrame +import graphics.scenery.utils.VideoDecoder +import graphics.scenery.volumes.DummyVolume +import graphics.scenery.volumes.RemoteRenderingProperties +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.TransferFunctionEditor +import graphics.scenery.volumes.vdi.VDIData +import graphics.scenery.volumes.vdi.VDINode +import graphics.scenery.volumes.vdi.VDIStreamer +import org.joml.Vector3f +import org.joml.Vector3i +import org.scijava.ui.behaviour.ClickBehaviour +import org.zeromq.ZContext +import java.nio.ByteBuffer +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.concurrent.thread + +/** + * An example client that can be used for remote volume rendering. Capable of switching between two visualization modes: + * receiving an encoded video stream and displaying it, and receiving Volumetric Depth Images (VDIs) and rendering them. + * + * Can be used with [ServerApplication]. + * + * @author Aryaman Gupta and Wissal Salhi + */ +class ClientApplication : SceneryBase("Client Application", 512, 512) { + + var buffer: ByteBuffer = ByteBuffer.allocateDirect(0) + val context = ZContext(4) + + val numSupersegments = 20 + var vdiStreaming = AtomicBoolean(true) + var firstVDIStream = true + + val displayPlane = FullscreenObject() + lateinit var vdiNode: VDINode + + val remoteRenderingProperties = RemoteRenderingProperties() + + override fun init() { + + //Step 1: Create necessary common components + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + val cam: Camera = DetachedHeadCamera() + with(cam) { + name = "ClientCamera" + spatial { + position = Vector3f(0.0f, 0.0f, 5.0f) + } + perspectiveCamera(50.0f, 512, 512) + scene.addChild(this) + } + cam.farPlaneDistance = 20.0f + + //Step 2: Create necessary video-streaming components + val dummyVolume = DummyVolume() + with(dummyVolume) { + name = "DummyVolume" + transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(this) + } + + with(displayPlane) { + name = "VRplane" + displayPlane.wantsSync = false + scene.addChild(this) + } + + val bridge = SwingBridgeFrame("1DTransferFunctionEditor") + val transferFunctionUI = TransferFunctionEditor(dummyVolume) + bridge.addPanel(transferFunctionUI) + transferFunctionUI.name = dummyVolume.name + val swingUiNode = bridge.uiNode + swingUiNode.spatial() { + position = Vector3f(2f, 0f, 0f) + } + + val vdiData = VDIData() + + //Step 3: Create vdi node and its properties + vdiNode = VDINode(windowWidth, windowHeight, numSupersegments, vdiData) + scene.addChild(vdiNode) + + //Attaching empty textures as placeholders for VDIs before actual VDIs arrive so that rendering can begin + vdiNode.attachEmptyTextures(VDINode.DoubleBuffer.First) + vdiNode.attachEmptyTextures(VDINode.DoubleBuffer.Second) + + vdiNode.skip_empty = false + + val VDIPlane = FullscreenObject() + with(VDIPlane) { + name = "VDIplane" + wantsSync = false + material().textures["diffuse"] = vdiNode.material().textures["OutputViewport"]!! + } + + //Step 4: add RemoteRenderingProperties Node to scene + scene.addChild(remoteRenderingProperties) + + vdiStreaming.set(false) + + remoteRenderingProperties.streamType = RemoteRenderingProperties.StreamType.VDI + var currentMode = RemoteRenderingProperties.StreamType.None + + val vdiStreamer = VDIStreamer() + + thread { + + while (!renderer!!.firstImageReady) { + Thread.sleep(50) + } + + //Step 5: switching code + renderer!!.runAfterRendering.add { + if (currentMode != RemoteRenderingProperties.StreamType.VolumeRendering + && remoteRenderingProperties.streamType == RemoteRenderingProperties.StreamType.VolumeRendering + ) { + + logger.info("Switching to Volume Rendering") + + vdiStreaming.set(false) + scene.addChild(displayPlane) + scene.removeChild(vdiNode) + scene.removeChild(VDIPlane) + + thread { + decodeVideo(displayPlane) + } + currentMode = RemoteRenderingProperties.StreamType.VolumeRendering + } else if (currentMode != RemoteRenderingProperties.StreamType.VDI + && remoteRenderingProperties.streamType == RemoteRenderingProperties.StreamType.VDI + ) { + + logger.info("Switching to VDI streaming") + + vdiStreaming.set(true) + scene.addChild(vdiNode) + scene.addChild(VDIPlane) + scene.removeChild(displayPlane) + + if (firstVDIStream) { + thread { + vdiStreamer.receiveAndUpdate( + vdiNode, + "tcp://localhost:6655", + renderer!!, + windowWidth, + windowHeight, + numSupersegments + ) + } + firstVDIStream = false + } + currentMode = RemoteRenderingProperties.StreamType.VDI + } + } + } + } + + private fun decodeVideo( plane: FullscreenObject){ + var decodedFrameCount: Int = 0 + val videoDecoder = VideoDecoder("scenery-stream.sdp") + logger.info("video decoder object created") + + while (!sceneInitialized()) { + Thread.sleep(200) + } + decodedFrameCount = 1 + logger.info("Decoding and displaying frames") + while (!(vdiStreaming.get()) && videoDecoder.nextFrameExists) { + val image = videoDecoder.decodeFrame() + if(image != null) { // image can be null, e.g. when the decoder encounters invalid information between frames + drawFrame(image, videoDecoder.videoWidth, videoDecoder.videoHeight, plane) + decodedFrameCount++ + } + } + decodedFrameCount -= 1 + videoDecoder.close() + logger.info("Done decoding and displaying $decodedFrameCount frames.") + } + + private fun drawFrame(tex: ByteArray, width: Int, height: Int, plane: FullscreenObject) { + if(buffer.capacity() == 0) { + buffer = BufferUtils.allocateByteAndPut(tex) + } else { + buffer.put(tex).flip() + } + plane.material { + textures["diffuse"] = Texture(Vector3i(width, height, 1), 4, contents = buffer, mipmap = true) + } + } + + override fun inputSetup() { + setupCameraModeSwitching() + + inputHandler?.addBehaviour("change_mode", + ClickBehaviour { _, _ -> + if(remoteRenderingProperties.streamType == RemoteRenderingProperties.StreamType.VolumeRendering) { + remoteRenderingProperties.streamType = RemoteRenderingProperties.StreamType.VDI + } else { + remoteRenderingProperties.streamType = RemoteRenderingProperties.StreamType.VolumeRendering + } + }) + inputHandler?.addKeyBinding("change_mode", "T") + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + ClientApplication().main() + } + } +} + + diff --git a/src/test/java/graphics/scenery/tests/applications/vdi/ServerApplication.kt b/src/test/java/graphics/scenery/tests/applications/vdi/ServerApplication.kt new file mode 100644 index 0000000000..bb9407e011 --- /dev/null +++ b/src/test/java/graphics/scenery/tests/applications/vdi/ServerApplication.kt @@ -0,0 +1,166 @@ +package graphics.scenery.tests.applications.vdi + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.controls.TrackedStereoGlasses +import graphics.scenery.volumes.* +import graphics.scenery.volumes.vdi.VDIStreamer +import graphics.scenery.volumes.vdi.VDIVolumeManager +import org.joml.Vector3f +import org.zeromq.ZContext +import java.nio.file.Paths +import kotlin.concurrent.thread + +/** + * An example application for a volume rendering server capable of performing both volume rendering and generation + * and streaming of Volumetric Depth Images (VDIs). When volume rendering is performed, an encoded video is streamed. + * + * [ClientApplication] can be used as a client with this application as server. + * + * @author Aryaman Gupta and Wissal Salhi + */ +class ServerApplication : SceneryBase("Volume Server Example", 512, 512) { + + var hmd: TrackedStereoGlasses? = null + val maxSupersegments = 20 + val context: ZContext = ZContext(4) + var firstVDI = true + + override fun init() { + //Step 1: Create common elements for VDI streaming and volumeRendering + renderer = hub.add( SceneryElement.Renderer, Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + if(!Settings().get("RemoteCamera",false)) { + val cam: Camera = DetachedHeadCamera() + with(cam) { + spatial { + position = Vector3f(0.0f, 0.0f, 5.0f) + } + perspectiveCamera(50.0f, 512, 512) + + scene.addChild(this) + } + } + + val volume = Volume.fromPathRaw(Paths.get(getDemoFilesPath() + "/volumes/box-iso/"), hub) + volume.name = "volume" + volume.colormap = Colormap.get("viridis") + volume.spatial { + position = Vector3f(0.0f, 0.0f, -3.5f) + rotation = rotation.rotateXYZ(0.05f, 0.05f, 0.05f) + scale = Vector3f(20.0f, 20.0f, 20.0f) + } + + volume.transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(volume) + + //Step 2: create a volume manager for vdi and add the volume to it: + var vdiVolumeManager = VDIVolumeManager(hub, windowWidth, windowHeight, maxSupersegments, scene).createVDIVolumeManager() + + vdiVolumeManager.customUniforms.add("doGeneration") + vdiVolumeManager.shaderProperties["doGeneration"] = true + + //Step 3: save the standard volume manger, the one that was first created with the volume + val standardVolumeManager: VolumeManager = hub.get() as VolumeManager + + renderer!!.runAfterRendering.add { + val dummyVolume = scene.find("DummyVolume") as? DummyVolume + if (dummyVolume != null) { + volume.transferFunction = dummyVolume.transferFunction + volume.maxDisplayRange = dummyVolume.maxDisplayRange + volume.minDisplayRange = dummyVolume.minDisplayRange + volume.colormap = dummyVolume.colormap + } + } + + //Step 4: switch between different modes + var currentMode = RemoteRenderingProperties.StreamType.None + + val vdiStreamer = VDIStreamer() + + thread { + + while (!renderer!!.firstImageReady) { + Thread.sleep(50) + } + + val volumeDimensions3i = Vector3f(volume.getDimensions().x.toFloat(), volume.getDimensions().y.toFloat(), volume.getDimensions().z.toFloat()) + + scene.findObserver()?.let { cam -> + vdiStreamer.setup("tcp://0.0.0.0:6655", cam, volumeDimensions3i, volume, maxSupersegments, vdiVolumeManager, renderer!!) + } + + renderer!!.runAfterRendering.add { + val switchMode = scene.find("RemoteRenderingProperties") as? RemoteRenderingProperties + + if (switchMode != null) { + logger.info("Value of switch mode: ${switchMode.streamType.toString()}") + + if (currentMode != RemoteRenderingProperties.StreamType.VolumeRendering + && switchMode.streamType == RemoteRenderingProperties.StreamType.VolumeRendering) { + + logger.info("Server will switch to Volume Rendering") + + vdiStreamer.vdiStreaming.set(false) + standardVolumeManager.replace(vdiVolumeManager) + startVideoStream() + + currentMode = RemoteRenderingProperties.StreamType.VolumeRendering + + } + else if (currentMode != RemoteRenderingProperties.StreamType.VDI + && switchMode.streamType == RemoteRenderingProperties.StreamType.VDI) { + //TODO: investigating glitching in VDI generation/streaming and comparing with VDIStreamingExample where they do not happen + logger.info("Server will switch to VDI Streaming") + + if(currentMode == RemoteRenderingProperties.StreamType.VolumeRendering) { + //stop the video streaming + renderer?.recordMovie() + } + vdiVolumeManager.replace(standardVolumeManager) + + vdiStreamer.vdiStreaming.set(true) + + currentMode = RemoteRenderingProperties.StreamType.VDI + } + else if (currentMode != RemoteRenderingProperties.StreamType.None + && switchMode.streamType == RemoteRenderingProperties.StreamType.None) { + + logger.info("Server will stop streaming") + + vdiStreamer.vdiStreaming.set(false) + if(currentMode == RemoteRenderingProperties.StreamType.VolumeRendering) { + renderer?.recordMovie() + } + + currentMode = RemoteRenderingProperties.StreamType.None + } + } + } + } + + } + + private fun startVideoStream() { + settings.set("VideoEncoder.StreamVideo", true) + settings.set("VideoEncoder.StreamingAddress", "rtp://127.0.0.2:5004") + renderer!!.recordMovie() + } + + override fun inputSetup() { + setupCameraModeSwitching() + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + ServerApplication().main() + } + } +} diff --git a/src/test/java/graphics/scenery/tests/applications/vdi/VDIClientExample.kt b/src/test/java/graphics/scenery/tests/applications/vdi/VDIClientExample.kt new file mode 100644 index 0000000000..b7adbcbf9d --- /dev/null +++ b/src/test/java/graphics/scenery/tests/applications/vdi/VDIClientExample.kt @@ -0,0 +1,89 @@ +package graphics.scenery.tests.applications.vdi + +import graphics.scenery.SceneryBase +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.volumes.vdi.VDIData +import graphics.scenery.volumes.vdi.VDINode +import graphics.scenery.volumes.vdi.VDIStreamer +import org.joml.Vector3f +import org.zeromq.ZContext +import kotlin.concurrent.thread + +/** + * Example application showing how to create a client application for receiving VDIs across a network + * and rendering them. + * + * To launch, the following VM parameter needs to be set: -Dscenery.Server=true + * + * Though this is a client application for streaming VDIs, it is a server in scenery's networking code + * as it controls the scene configuration (e.g., camera pose). + * + * Can be used with [VDIStreamingExample] + * + * @author Aryaman Gupta + */ +class VDIClientExample : SceneryBase("VDI Client", 512, 512, wantREPL = false) { + + val cam: Camera = DetachedHeadCamera() + val plane = FullscreenObject() + val context = ZContext(4) + val numSupersegments = 20 + + lateinit var vdiNode: VDINode + val skipEmpty = true + + override fun init() { + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + //Step1: create Camera + with(cam) { + spatial { + position = Vector3f(0.0f, 0.5f, 5.0f) + } + perspectiveCamera(50.0f, windowWidth, windowHeight) + scene.addChild(this) + } + cam.farPlaneDistance = 20.0f + + val vdiData = VDIData() + + //Step 2: Create vdi node and its properties + vdiNode = VDINode(windowWidth, windowHeight, numSupersegments, vdiData) + scene.addChild(vdiNode) + + //Attaching empty textures as placeholders for VDIs before actual VDIs arrive so that rendering can begin + vdiNode.attachEmptyTextures(VDINode.DoubleBuffer.First) + vdiNode.attachEmptyTextures(VDINode.DoubleBuffer.Second) + + vdiNode.skip_empty = false + vdiNode.visible = true + + //Step3: set plane properties + scene.addChild(plane) + plane.material().textures["diffuse"] = vdiNode.material().textures["OutputViewport"]!! + + val vdiStreamer = VDIStreamer() + + //Step 4: call receive and update VDI + thread { + while (!renderer!!.firstImageReady) { + Thread.sleep(100) + } + vdiStreamer.receiveAndUpdate(vdiNode, "tcp://localhost:6655", renderer!!, windowWidth, windowHeight, numSupersegments) + } + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + VDIClientExample().main() + } + } +} diff --git a/src/test/java/graphics/scenery/tests/applications/vdi/VDIStreamingExample.kt b/src/test/java/graphics/scenery/tests/applications/vdi/VDIStreamingExample.kt new file mode 100644 index 0000000000..7c810562b5 --- /dev/null +++ b/src/test/java/graphics/scenery/tests/applications/vdi/VDIStreamingExample.kt @@ -0,0 +1,103 @@ +package graphics.scenery.tests.applications.vdi + +import graphics.scenery.Camera +import graphics.scenery.DetachedHeadCamera +import graphics.scenery.SceneryBase +import graphics.scenery.Settings +import graphics.scenery.backends.Renderer +import graphics.scenery.volumes.* +import graphics.scenery.volumes.vdi.VDIStreamer +import graphics.scenery.volumes.vdi.VDIVolumeManager +import org.joml.Vector3f +import java.nio.file.Paths +import kotlin.concurrent.thread + +/** + * Example showing how Volumetric Depth Images (VDIs) can be generated and streamed across a network. + * + * To launch, the following VM parameters need to be set: + * -Dscenery.ServerAddress=, e.g., -Dscenery.ServerAddress=tcp://127.0.0.1 + * -Dscenery.RemoteCamera=true + * + * Though this is a server application for streaming VDIs, it is a client in scenery's networking code + * as it obtains the scene configuration (e.g., camera pose) from the VDI streaming client. + * + * Can be used with [VDIClientExample] + * + * @author Aryaman Gupta and Wissal Salhi + */ +class VDIStreamingExample : SceneryBase("VDI Streaming Example", 512, 512) { + + val cam: Camera = DetachedHeadCamera() + + val maxSupersegments = 20 + override fun init() { + + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + //Step 1: create necessary components: camera, volume, volumeManager + if(!Settings().get("RemoteCamera",false)) { + with(cam) { + spatial { + position = Vector3f(0.0f, 0.5f, 5.0f) + } + perspectiveCamera(50.0f, windowWidth, windowHeight) + scene.addChild(this) + } + } + + val volume = Volume.fromPathRaw(Paths.get(getDemoFilesPath() + "/volumes/box-iso/"), hub) + volume.name = "volume" + volume.colormap = Colormap.get("viridis") + volume.spatial { + position = Vector3f(0.0f, 0.0f, -3.5f) + rotation = rotation.rotateXYZ(0.05f, 0.05f, 0.05f) + scale = Vector3f(20.0f, 20.0f, 20.0f) + } + volume.transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(volume) + + // Step 2: Create VDI Volume Manager + val vdiVolumeManager = VDIVolumeManager(hub, windowWidth, windowHeight, maxSupersegments, scene).createVDIVolumeManager() + + //step 3: switch the volume's current volume manager to VDI volume manager + volume.volumeManager = vdiVolumeManager + + // Step 4: add the volume to VDI volume manager + vdiVolumeManager.add(volume) + volume.volumeManager.shaderProperties["doGeneration"] = true + + // Step 5: add the VDI volume manager to the hub + hub.add(vdiVolumeManager) + + //Step 6: transmitting the VDI + val volumeDimensions = Vector3f(volume.getDimensions().x.toFloat(),volume.getDimensions().y.toFloat(),volume.getDimensions().z.toFloat()) + + val vdiStreamer = VDIStreamer() + + thread { + while (!renderer!!.firstImageReady) { + Thread.sleep(50) + } + + vdiStreamer.vdiStreaming.set(true) + vdiStreamer.setup("tcp://0.0.0.0:6655", scene.findObserver()!!, volumeDimensions, volume, maxSupersegments, vdiVolumeManager, + renderer!! + ) + } + + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + VDIStreamingExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/applications/volumes/SimpleVolumeClient.kt b/src/test/kotlin/graphics/scenery/tests/applications/volumes/SimpleVolumeClient.kt new file mode 100644 index 0000000000..cab486ddbb --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/applications/volumes/SimpleVolumeClient.kt @@ -0,0 +1,109 @@ +package graphics.scenery.tests.applications.volumes + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.textures.Texture +import graphics.scenery.ui.SwingBridgeFrame +import graphics.scenery.utils.VideoDecoder +import graphics.scenery.volumes.DummyVolume +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.TransferFunctionEditor +import org.joml.Vector3f +import org.joml.Vector3i +import java.nio.ByteBuffer +import kotlin.concurrent.thread + +/** + * Simple client application for remote server-side volume rendering. Receives and displays a video stream from + * the server (e.g. [SimpleVolumeServerExample]). Uses a [DummyVolume] to capture changes in volume rendering + * parameters that are synchronized with the server. Camera is also synchronized with the server. + * + * Start with vm param: + * -Dscenery.Server=true + * + * Explanation: + * This application, the client in the remote volume rendering setup, is the server in scenery's networking code + * because the camera and volume properties from this scene need to be used in the remote rendering server. + */ +class SimpleVolumeClient : SceneryBase("Volume Client", 512, 512) { + + val displayPlane = FullscreenObject() + var buffer: ByteBuffer = ByteBuffer.allocateDirect(0) + + override fun init() { + + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + val cam: Camera = DetachedHeadCamera() + with(cam) { + name = "ClientCamera" + spatial { + position = Vector3f(0.0f, 0.0f, 5.0f) + } + perspectiveCamera(50.0f, 512, 512) + wantsSync = true + scene.addChild(this) + } + + val dummyVolume = DummyVolume() + with(dummyVolume) { + name = "DummyVolume" + transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(this) + } + + val bridge = SwingBridgeFrame("1DTransferFunctionEditor") + val tfUI = TransferFunctionEditor(dummyVolume) + bridge.addPanel(tfUI) + tfUI.name = dummyVolume.name + val swingUiNode = bridge.uiNode + swingUiNode.spatial() { + position = Vector3f(2f, 0f, 0f) + } + + with(displayPlane){ + name = "plane" + wantsSync = false + scene.addChild(this) + } + + val videoDecoder = VideoDecoder("scenery-stream.sdp") + thread { + while(!renderer!!.firstImageReady) { + Thread.sleep(50) + } + + videoDecoder.decodeFrameByFrame(drawFrame) + } + } + + private val drawFrame: (ByteArray, Int, Int, Int) -> Unit = {tex: ByteArray, width: Int, height: Int, frameIndex: Int -> + if(frameIndex % 100 == 0) { + logger.debug("Displaying frame $frameIndex") + } + + if(buffer.capacity() == 0) { + buffer = BufferUtils.allocateByteAndPut(tex) + } else { + buffer.put(tex).flip() + } + + displayPlane.material { + textures["diffuse"] = Texture(Vector3i(width, height, 1), 4, contents = buffer, mipmap = true) + } + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + SimpleVolumeClient().main() + } + } + +} \ No newline at end of file diff --git a/src/test/kotlin/graphics/scenery/tests/applications/volumes/SimpleVolumeServerExample.kt b/src/test/kotlin/graphics/scenery/tests/applications/volumes/SimpleVolumeServerExample.kt new file mode 100644 index 0000000000..dd2e5a33ce --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/applications/volumes/SimpleVolumeServerExample.kt @@ -0,0 +1,97 @@ +package graphics.scenery.tests.applications.volumes + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.DummyVolume +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import org.joml.Vector3f +import java.nio.file.Paths +import kotlin.concurrent.thread + +/** + * Simple server application for remote server-side volume rendering. Renders a volume and transmits the rendered + * image as an encoded video stream. Obtains volume rendering parameters and camera by synchronizing with the + * remote client. + * + * Example client: [SimpleVolumeClient] + * + * Start master with vm param: + * -Dscenery.ServerAddress={client's IP address} -Dscenery.RemoteCamera=true + * + * Explanation: + * This application, the server in the remote volume rendering setup, is the client in scenery's networking code + * because it uses camera and volume rendering parameters from the remote client. + */ +class SimpleVolumeServerExample : SceneryBase("Volume Server Example", 512, 512) { + + override fun init() { + renderer = hub.add( + SceneryElement.Renderer, + Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight) + ) + + if(!Settings().get("RemoteCamera", false)) { + val cam: Camera = DetachedHeadCamera() + with(cam) { + spatial { + position = Vector3f(0.0f, 0.0f, 5.0f) + } + perspectiveCamera(50.0f, 512, 512) + scene.addChild(this) + } + } + + val volume = Volume.fromPathRaw(Paths.get(getDemoFilesPath() + "/volumes/box-iso/"), hub) + volume.name = "volume" + volume.colormap = Colormap.get("viridis") + volume.spatial { + position = Vector3f(0.0f, 0.0f, -3.5f) + rotation = rotation.rotateXYZ(0.05f, 0.05f, 0.05f) + scale = Vector3f(20.0f, 20.0f, 20.0f) + } + volume.transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(volume) + + settings.set("VideoEncoder.StreamVideo", true) + settings.set("VideoEncoder.StreamingAddress", "rtp://127.0.0.2:5004") + renderer!!.recordMovie() + + renderer!!.runAfterRendering.add { + val dummyVolume = scene.find("DummyVolume") as? DummyVolume + if (dummyVolume != null) { + volume.transferFunction = dummyVolume.transferFunction + volume.maxDisplayRange = dummyVolume.maxDisplayRange + volume.minDisplayRange = dummyVolume.minDisplayRange + volume.colormap = dummyVolume.colormap + } + } + + thread { + while(true) { + volume.spatial { + rotation = rotation.rotateY(0.003f) + } + Thread.sleep(5) + } + } + } + + override fun inputSetup() { + setupCameraModeSwitching() + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + SimpleVolumeServerExample().main() + } + } +} \ No newline at end of file diff --git a/src/test/kotlin/graphics/scenery/tests/examples/advanced/AsyncTextureExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/advanced/AsyncTextureExample.kt new file mode 100644 index 0000000000..5bd969dfdf --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/advanced/AsyncTextureExample.kt @@ -0,0 +1,170 @@ +package graphics.scenery.tests.examples.advanced + +import graphics.scenery.* +import org.joml.Vector3f +import graphics.scenery.backends.Renderer +import graphics.scenery.attribute.material.Material +import graphics.scenery.compute.ComputeMetadata +import graphics.scenery.compute.InvocationType +import graphics.scenery.primitives.Plane +import graphics.scenery.textures.Texture +import graphics.scenery.textures.UpdatableTexture +import graphics.scenery.utils.Image +import graphics.scenery.utils.RingBuffer +import graphics.scenery.volumes.Volume +import net.imglib2.type.numeric.integer.UnsignedByteType +import org.joml.Vector3i +import org.lwjgl.system.MemoryUtil +import kotlin.concurrent.thread +import kotlin.system.measureTimeMillis +import kotlin.time.Duration.Companion.nanoseconds + +/** + * Example loading a large texture asynchronously + * + * @author Ulrik Günther + */ +class AsyncTextureExample: SceneryBase("Async Texture example", 512, 512) { + lateinit var volume: Volume + private val size = Vector3i(windowWidth,windowHeight,512) + var previous = 0L + var frametimes = ArrayList() + + override fun init() { + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + renderer?.runAfterRendering?.add { + val now = System.nanoTime() + val duration = (now - previous).nanoseconds + previous = now + + frametimes.add(duration.inWholeMilliseconds.toInt()) + } + + val cam: Camera = DetachedHeadCamera() + with(cam) { + perspectiveCamera(50.0f, windowWidth, windowHeight) + spatial { + position = Vector3f(0.0f, 0.0f, 5.0f) + } + scene.addChild(this) + } + + val b = Box(Vector3f(0.5f)) + b.material().diffuse = Vector3f(0.5f) + scene.addChild(b) + + val a = AmbientLight(1.0f) + scene.addChild(a) + + val p = Plane(Vector3f(1000.0f, 1000.0f, 0.01f)) + p.material().diffuse = Vector3f(0.0f, 1.0f, 0.0f) + p.material().cullingMode = Material.CullingMode.None + p.material().depthTest = true + p.material().depthOp = Material.DepthTest.LessEqual + p.spatial().position = Vector3f(0.0f, 0.0f, -1000.0f) + cam.addChild(p) + + Light.createLightTetrahedron(spread = 4.0f, radius = 15.0f, intensity = 0.5f) + .forEach { scene.addChild(it) } + + val buffer = MemoryUtil.memCalloc(windowWidth * windowHeight * 4) + + val compute = RichNode() + compute.name = "compute node" + val computeTexture = Texture.fromImage( + Image(buffer, windowWidth, windowHeight, 1), + usage = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + compute.setMaterial(ShaderMaterial.fromFiles(this::class.java, "CheckDataForAsyncExample.comp")) { + textures["OutputViewport"] = computeTexture + } + compute.metadata["ComputeMetadata"] = ComputeMetadata( + workSizes = Vector3i(windowWidth, windowHeight, 1), + invocationType = InvocationType.Permanent + ) + + scene.addChild(compute) + + val plane = FullscreenObject() + plane.material().textures["diffuse"] = compute.material().textures["OutputViewport"]!! + scene.addChild(plane) + + // We create textures and backing buffers separately, + // as UpdatableTexture are supposed to have contents = null at the moment + val textures = RingBuffer(2, cleanup = null, default = { + UpdatableTexture( + size, + channels = 1, + type = UnsignedByteType(), + usageType = hashSetOf(Texture.UsageType.Texture, Texture.UsageType.AsyncLoad, Texture.UsageType.LoadStoreImage), + contents = null + ) + }) + + val backing = RingBuffer(2, cleanup = null, default = { + val mem = MemoryUtil.memAlloc(size.x*size.y*size.z) + while (mem.remaining() > 0) { + mem.put((it * 63).toByte()) + } + mem.flip() + mem + }) + + thread(isDaemon = true) { + Thread.sleep(5000) + + while(running) { + val index = textures.currentReadPosition + val texture = textures.get() + + logger.info("Fiddling Permits available: ${texture.mutex.availablePermits()}") + logger.info("Upload Permits available: ${texture.gpuMutex.availablePermits()}") + Thread.sleep(50) + + // We add a TextureUpdate that covers the whole texture, + // using one of the backing RingBuffers. + val update = UpdatableTexture.TextureUpdate( + UpdatableTexture.TextureExtents(0, 0, 0, size.x, size.y, size.z), + backing.get() + ) + texture.addUpdate(update) + + // Reassigning the texture here, together with its one update + compute.material().textures["humongous"] = texture + + val waitTime = measureTimeMillis { + // Here, we wait until the texture is marked as available on the GPU + while(!texture.availableOnGPU() && running) { + logger.info("Texture $index not available yet, uploaded=${texture.uploaded.get()}/permits=${texture.gpuMutex.availablePermits()}") + Thread.sleep(10) + } + } + + logger.info("Texture $index is available now, waited $waitTime ms") + + // After the texture is available, we proceed to the next texture + // in the RingBuffer, and reset the current texture's uploaded + // AtomicInteger to 0 + texture.uploaded.set(0) + + Thread.sleep(500) + } + } + } + + override fun inputSetup() { + setupCameraModeSwitching() + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + AsyncTextureExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/advanced/RunAfterRenderExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/advanced/RunAfterRenderExample.kt new file mode 100644 index 0000000000..282b4d24a7 --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/advanced/RunAfterRenderExample.kt @@ -0,0 +1,104 @@ +package graphics.scenery.tests.examples.advanced + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.tests.examples.basic.TexturedCubeExample +import graphics.scenery.textures.Texture +import graphics.scenery.utils.Image +import org.joml.Vector3f +import kotlin.concurrent.thread +import kotlin.test.assertEquals + +/** + * Example to show how run after rendering lambdas may be used to produce animations that are + * synchronized with the render loop + * + * @author Aryaman Gupta + */ +class RunAfterRenderExample : SceneryBase("RunAfterRenderExample") { + + private val boxRotation = Vector3f(0.0f) + private var totalFrames = 0L + private val quantumOfRotation = 0.01f + + override fun init() { + renderer = hub.add( + SceneryElement.Renderer, + Renderer.createRenderer(hub, applicationName, scene, 700, 700) + ) + + val box = Box(Vector3f(1.0f, 1.0f, 1.0f)) + box.name = "le box du win" + box.material().textures["diffuse"] = Texture.fromImage(Image.fromResource("textures/helix.png", TexturedCubeExample::class.java)) + box.material().metallic = 0.3f + box.material().roughness = 0.9f + scene.addChild(box) + + val light = PointLight(radius = 15.0f) + light.spatial().position = Vector3f(0.0f, 0.0f, 2.0f) + light.intensity = 5.0f + light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + scene.addChild(light) + + val cam: Camera = DetachedHeadCamera() + with(cam) { + spatial().position = Vector3f(0.0f, 0.0f, 5.0f) + perspectiveCamera(50.0f, 512, 512) + + scene.addChild(this) + } + + renderer?.runAfterRendering?.add { + box.spatial().rotation.rotateY(quantumOfRotation) + box.spatial().needsUpdate = true + logger.info("Initial rot: ${box.rotation}") + } + + thread { + while (renderer?.firstImageReady == false) { + Thread.sleep(5) + } + + Thread.sleep(1000) //give some time for the rendering to take place + + renderer?.close() + Thread.sleep(200) //give some time for the renderer to close + + box.spatial().rotation.getEulerAnglesXYZ(boxRotation) + totalFrames = renderer?.totalFrames!! + } + } + + override fun main() { + // add assertions, these only get called when the example is called + // as part of scenery's integration tests + assertions[AssertionCheckPoint.AfterClose]?.add { + val testBox = Box(Vector3f(1.0f, 1.0f, 1.0f)) + + var cnt = 0 + while(cnt) { + RunAfterRenderExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/basic/AtmosphereExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/basic/AtmosphereExample.kt new file mode 100644 index 0000000000..b5d634dca3 --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/basic/AtmosphereExample.kt @@ -0,0 +1,105 @@ +package graphics.scenery.tests.examples.basic + +import bdv.util.AxisOrder +import bvv.core.VolumeViewerOptions +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.controls.OpenVRHMD +import graphics.scenery.numerics.Random +import graphics.scenery.primitives.Atmosphere +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import ij.IJ +import ij.ImagePlus +import net.imglib2.img.Img +import net.imglib2.img.display.imagej.ImageJFunctions +import net.imglib2.type.numeric.integer.UnsignedShortType +import org.joml.Vector3f + +/** + * + * A basic example that shows how the atmosphere object can be applied to a scene. + * Per default, the sun position is synchronized to the local time. + * The sun position can be changed with Ctrl + arrow keys and fine-tuned with Shift-Ctrl + arrow keys. + * Call [Atmosphere.attachRotateBehaviours] in the input setup to add sun controls. + * @author Samuel Pantze + */ +class AtmosphereExample : SceneryBase("Atmosphere Example", + windowWidth = 1024, windowHeight = 768) { + + /** Whether to run this example in VR mode. */ + private val useVR = false + + lateinit var volume: Volume + + private var atmos = Atmosphere(emissionStrength = 0.3f) + + private lateinit var hmd: OpenVRHMD + + override fun init() { + + renderer = hub.add( + SceneryElement.Renderer, + Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + if (useVR) { + hmd = OpenVRHMD(useCompositor = true) + hub.add(SceneryElement.HMDInput, hmd) + renderer?.toggleVR() + } + + val imp: ImagePlus = IJ.openImage("https://imagej.nih.gov/ij/images/t1-head.zip") + val img: Img = ImageJFunctions.wrapShort(imp) + + volume = Volume.fromRAI(img, UnsignedShortType(), AxisOrder.DEFAULT, "T1 head", hub, VolumeViewerOptions()) + volume.colormap = Colormap.get("jet") + volume.setTransferFunctionRange(10f, 1000f) + volume.transferFunction = TransferFunction.ramp(0.001f, 0.2f, 1f) + volume.spatial().scale = Vector3f(1f, 1f, 2.3f) + scene.addChild(volume) + + val ambientLight = AmbientLight(0.1f) + scene.addChild(ambientLight) + + val lights = (1 until 5).map { + val light = PointLight(10f) + val spread = 2f + light.spatial().position = Vector3f( + Random.randomFromRange(-spread, spread), + Random.randomFromRange(-spread, spread), + Random.randomFromRange(-spread, spread), + ) + light.intensity = 1f + scene.addChild(light) + light + } + + val cam = if (useVR) {DetachedHeadCamera(hmd)} else {DetachedHeadCamera()} + with(cam) { + spatial { + position = Vector3f(0.0f, 0.0f, 5.0f) + } + perspectiveCamera(70.0f, 512, 768) + scene.addChild(this) + } + + scene.addChild(atmos) + } + + override fun inputSetup() { + super.inputSetup() + + setupCameraModeSwitching() + + inputHandler?.let { atmos.attachRotateBehaviours(it) } + } + + companion object { + /** AtmosphereExample main function */ + @JvmStatic + fun main(args: Array) { + AtmosphereExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/cluster/ClusterExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/cluster/ClusterExample.kt index 13f0bae87f..e7796ae544 100644 --- a/src/test/kotlin/graphics/scenery/tests/examples/cluster/ClusterExample.kt +++ b/src/test/kotlin/graphics/scenery/tests/examples/cluster/ClusterExample.kt @@ -1,25 +1,19 @@ package graphics.scenery.tests.examples.cluster -import org.joml.Vector3f import graphics.scenery.* import graphics.scenery.backends.Renderer -import graphics.scenery.controls.InputHandler import graphics.scenery.controls.TrackedStereoGlasses -import graphics.scenery.net.NodePublisher -import graphics.scenery.net.NodeSubscriber import graphics.scenery.numerics.Random import graphics.scenery.primitives.Cone import graphics.scenery.proteins.Protein import graphics.scenery.proteins.RibbonDiagram -import graphics.scenery.utils.Statistics import graphics.scenery.utils.extensions.minus import graphics.scenery.volumes.BufferedVolume import graphics.scenery.volumes.Colormap import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume -import org.scijava.ui.behaviour.ClickBehaviour +import org.joml.Vector3f import java.nio.file.Paths -import kotlin.concurrent.thread import kotlin.math.PI import kotlin.math.floor import kotlin.math.roundToInt diff --git a/src/test/kotlin/graphics/scenery/tests/examples/cluster/GiovannisExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/cluster/GiovannisExample.kt index 9a4684687f..3515de36b2 100644 --- a/src/test/kotlin/graphics/scenery/tests/examples/cluster/GiovannisExample.kt +++ b/src/test/kotlin/graphics/scenery/tests/examples/cluster/GiovannisExample.kt @@ -1,30 +1,21 @@ package graphics.scenery.tests.examples.cluster -import org.joml.Vector3f import graphics.scenery.* import graphics.scenery.backends.Renderer -import graphics.scenery.controls.InputHandler import graphics.scenery.controls.TrackedStereoGlasses import graphics.scenery.controls.behaviours.GamepadClickBehaviour import graphics.scenery.controls.behaviours.GamepadRotationControl -import graphics.scenery.net.NodePublisher -import graphics.scenery.net.NodeSubscriber -import graphics.scenery.numerics.Random import graphics.scenery.proteins.Protein import graphics.scenery.proteins.RibbonDiagram -import graphics.scenery.utils.Statistics -import graphics.scenery.utils.extensions.minus import graphics.scenery.volumes.BufferedVolume import graphics.scenery.volumes.Colormap import graphics.scenery.volumes.TransferFunction import graphics.scenery.volumes.Volume import net.java.games.input.Component -import org.scijava.ui.behaviour.ClickBehaviour +import org.joml.Vector3f import java.nio.file.Paths import kotlin.concurrent.thread import kotlin.math.PI -import kotlin.math.floor -import kotlin.math.roundToInt /** * Example to demonstrate rendering on a cluster. Will display a grid of geometric options, diff --git a/src/test/kotlin/graphics/scenery/tests/examples/compute/CustomVolumeManagerExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/compute/CustomVolumeManagerExample.kt index 5b3ca83720..28407b3046 100644 --- a/src/test/kotlin/graphics/scenery/tests/examples/compute/CustomVolumeManagerExample.kt +++ b/src/test/kotlin/graphics/scenery/tests/examples/compute/CustomVolumeManagerExample.kt @@ -8,8 +8,10 @@ import org.joml.Vector3f import graphics.scenery.* import graphics.scenery.backends.Renderer import graphics.scenery.textures.Texture +import graphics.scenery.ui.SwingBridgeFrame import graphics.scenery.utils.Image import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.TransferFunctionEditor import graphics.scenery.volumes.Volume import graphics.scenery.volumes.VolumeManager import ij.IJ @@ -18,6 +20,7 @@ import net.imglib2.img.Img import net.imglib2.img.display.imagej.ImageJFunctions import net.imglib2.type.numeric.integer.UnsignedShortType import org.lwjgl.system.MemoryUtil +import java.io.File /** * Example showing using a custom [graphics.scenery.volumes.VolumeManager] with @@ -36,7 +39,7 @@ class CustomVolumeManagerExample : SceneryBase("CustomVolumeManagerExample") { SegmentType.FragmentShader to SegmentTemplate( this.javaClass, "ComputeVolume.comp", - "intersectBoundingBox", "vis", "SampleVolume", "Convert", "Accumulate"), + "intersectBoundingBox", "vis", "localNear", "localFar", "SampleVolume", "Convert", "Accumulate"), )) volumeManager.customTextures.add("OutputRender") @@ -50,7 +53,15 @@ class CustomVolumeManagerExample : SceneryBase("CustomVolumeManagerExample") { val img: Img = ImageJFunctions.wrapShort(imp) val volume = Volume.fromRAI(img, UnsignedShortType(), AxisOrder.DEFAULT, "T1 head", hub, VolumeViewerOptions()) - volume.transferFunction = TransferFunction.ramp(0.001f, 0.5f, 0.3f) + + //transfer function and display range chosen empirically + volume.minDisplayRange = 0f + volume.maxDisplayRange = 1128.0f + volume.transferFunction.addControlPoint(0.0f, 0.0f) + volume.transferFunction.addControlPoint(0.12f, 0.3f) + volume.transferFunction.addControlPoint(0.30f, 0.0f) + volume.transferFunction.addControlPoint(1.0f, 0.0f) + scene.addChild(volume) val box = Box(Vector3f(1.0f, 1.0f, 1.0f)) diff --git a/src/test/kotlin/graphics/scenery/tests/examples/compute/PersistentTextureRequestsExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/compute/PersistentTextureRequestsExample.kt new file mode 100644 index 0000000000..eee669bb9a --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/compute/PersistentTextureRequestsExample.kt @@ -0,0 +1,152 @@ +package graphics.scenery.tests.examples.compute + +import bdv.util.AxisOrder +import bvv.core.VolumeViewerOptions +import bvv.core.shadergen.generate.SegmentTemplate +import bvv.core.shadergen.generate.SegmentType +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.backends.vulkan.VulkanRenderer +import graphics.scenery.textures.Texture +import graphics.scenery.utils.Image +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.VolumeManager +import ij.IJ +import ij.ImagePlus +import net.imglib2.img.Img +import net.imglib2.img.display.imagej.ImageJFunctions +import net.imglib2.type.numeric.integer.UnsignedShortType +import org.joml.Vector3f +import org.lwjgl.system.MemoryUtil +import java.util.concurrent.atomic.AtomicInteger +import kotlin.concurrent.thread +import kotlin.test.assertEquals + +/** + * Example to show how persistent texture requests - that are served once per frame - may be created + * and accessed. These may be particularly be useful, e.g, when compute shaders are used and output + * textures need to be accessed every time they are updated. + * + * @author Aryaman Gupta + * @author Ulrik Günther + */ +class PersistentTextureRequestsExample : SceneryBase("PersistentTextureRequestsExample") { + + private val counter = AtomicInteger(0) + private var totalFrames = -1L + + override fun init() { + renderer = hub.add( + SceneryElement.Renderer, + Renderer.createRenderer(hub, applicationName, scene, 1280, 720)) + + val volumeManager = VolumeManager(hub, + useCompute = true, + customSegments = hashMapOf( + SegmentType.FragmentShader to SegmentTemplate( + this.javaClass, + "ComputeVolume.comp", + "intersectBoundingBox", "vis", "localNear", "localFar", "SampleVolume", "Convert", "Accumulate"), + )) + volumeManager.customTextures.add("OutputRender") + + val outputBuffer = MemoryUtil.memCalloc(1280*720*4) + val outputTexture = Texture.fromImage(Image(outputBuffer, 1280, 720), usage = hashSetOf(Texture.UsageType.LoadStoreImage, Texture.UsageType.Texture)) + volumeManager.material().textures["OutputRender"] = outputTexture + + hub.add(volumeManager) + + val imp: ImagePlus = IJ.openImage("https://imagej.nih.gov/ij/images/t1-head.zip") + val img: Img = ImageJFunctions.wrapShort(imp) + + val volume = Volume.fromRAI(img, UnsignedShortType(), AxisOrder.DEFAULT, "T1 head", hub, VolumeViewerOptions()) + volume.transferFunction = TransferFunction.ramp(0.001f, 0.5f, 0.3f) + scene.addChild(volume) + + val box = Box(Vector3f(1.0f, 1.0f, 1.0f)) + box.name = "le box du win" + box.material { + textures["diffuse"] = outputTexture + metallic = 0.0f + roughness = 1.0f + } + + scene.addChild(box) + + val light = PointLight(radius = 15.0f) + light.spatial().position = Vector3f(0.0f, 0.0f, 2.0f) + light.intensity = 5.0f + light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + scene.addChild(light) + + val cam: Camera = DetachedHeadCamera() + with(cam) { + spatial().position = Vector3f(0.0f, 0.0f, 5.0f) + perspectiveCamera(50.0f, 512, 512) + + scene.addChild(this) + } + + thread { + val opTexture = volumeManager.material().textures["OutputRender"]!! + + var prevCounter = 0 + + (renderer as VulkanRenderer)?.persistentTextureRequests?.add(opTexture to counter) + + // this is the loop where you may do your tasks that are asynchronous to rendering + while (renderer?.shouldClose == false) { + while (counter.get() == prevCounter) { + Thread.sleep(5) + } + prevCounter = counter.get() + + //updated texture has been returned + val buffer = opTexture.contents + //the buffer can now, e.g., be transmitted, as is required for parallel rendering + + if (buffer != null && prevCounter == 100) { + SystemHelpers.dumpToFile(buffer, "texture-${SystemHelpers.formatDateTime(delimiter = "_")}.raw") + } + } + } + + thread { + while (renderer?.firstImageReady == false) { + Thread.sleep(5) + } + + Thread.sleep(1000) //give some time for the rendering to take place + + renderer?.close() + Thread.sleep(200) //give some time for the renderer to close + + totalFrames = renderer?.totalFrames!! + } + } + + override fun main() { + // add assertions, these only get called when the example is called + // as part of scenery's integration tests + assertions[AssertionCheckPoint.AfterClose]?.add { + + assertEquals ( counter.get().toLong(), totalFrames, "One texture was returned per render frame" ) + } + super.main() + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + PersistentTextureRequestsExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/stresstests/ExampleRunner.kt b/src/test/kotlin/graphics/scenery/tests/examples/stresstests/ExampleRunner.kt index 8e3b34675c..05f0737ebb 100644 --- a/src/test/kotlin/graphics/scenery/tests/examples/stresstests/ExampleRunner.kt +++ b/src/test/kotlin/graphics/scenery/tests/examples/stresstests/ExampleRunner.kt @@ -3,6 +3,7 @@ package graphics.scenery.tests.examples.stresstests import graphics.scenery.SceneryBase import graphics.scenery.SceneryElement import graphics.scenery.backends.Renderer +import graphics.scenery.numerics.Random import graphics.scenery.utils.ExtractsNatives import graphics.scenery.utils.lazyLogger import graphics.scenery.utils.SystemHelpers @@ -17,9 +18,7 @@ import kotlin.system.exitProcess import kotlin.test.assertFalse import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.minutes -import kotlin.time.ExperimentalTime -@OptIn(ExperimentalTime::class) @RunWith(Parameterized::class) class ExampleRunner( private val clazz: Class<*>, @@ -44,60 +43,59 @@ class ExampleRunner( val instance: SceneryBase = clazz.getConstructor().newInstance() as SceneryBase var failure = false - try { - val handler = CoroutineExceptionHandler { _, e -> - logger.error("${clazz.simpleName}: Received exception $e") - logger.error("Stack trace: ${e.stackTraceToString()}") + val handler = CoroutineExceptionHandler { _, e -> + logger.error("${clazz.simpleName}: Received exception $e") + logger.error("Stack trace: ${e.stackTraceToString()}") - failure = true - // we fail very hard here to prevent process clogging the CI - exitProcess(-1) - } - - val exampleRunnable = GlobalScope.launch(handler) { - instance.assertions[SceneryBase.AssertionCheckPoint.BeforeStart]?.forEach { - it.invoke() - } - instance.main() - } + failure = true + // we fail very hard here to prevent process clogging the CI + exitProcess(-1) + } - while (!instance.running || !instance.sceneInitialized() || instance.hub.get(SceneryElement.Renderer) == null) { - delay(200) - } - val r = (instance.hub.get(SceneryElement.Renderer) as Renderer) + // re-seed scenery's PRNG to the seed given via system property + Random.reseed() - while(!r.firstImageReady) { - delay(200) + val exampleRunnable = GlobalScope.launch(handler) { + instance.assertions[SceneryBase.AssertionCheckPoint.BeforeStart]?.forEach { + it.invoke() } + instance.main() + } - delay(2000) - r.screenshot("$rendererDirectory/${clazz.simpleName}.png", overwrite = true) - Thread.sleep(2000) + while (!instance.running || !instance.sceneInitialized() || instance.hub.get(SceneryElement.Renderer) == null) { + delay(200) + } + val r = (instance.hub.get(SceneryElement.Renderer) as Renderer) - logger.info("Sending close to ${clazz.simpleName}") - instance.close() - instance.assertions[SceneryBase.AssertionCheckPoint.AfterClose]?.forEach { - it.invoke() - } + while(!r.firstImageReady) { + delay(200) + } - while (instance.running && !failure) { - if (runtime > maxRuntimePerTest) { - exampleRunnable.cancelAndJoin() - logger.error("Maximum runtime of $maxRuntimePerTest exceeded, aborting test run.") - failure = true - } + delay(3000) + r.screenshot("$rendererDirectory/${clazz.simpleName}.png", overwrite = true) + Thread.sleep(2000) - runtime += 200.milliseconds - delay(200) - } + logger.info("Sending close to ${clazz.simpleName}") + instance.close() + instance.assertions[SceneryBase.AssertionCheckPoint.AfterClose]?.forEach { + it.invoke() + } - if(failure) { + while (instance.running && !failure) { + if (runtime > maxRuntimePerTest) { exampleRunnable.cancelAndJoin() - } else { - exampleRunnable.join() + logger.error("Maximum runtime of $maxRuntimePerTest exceeded, aborting test run.") + failure = true } - } catch (e: ThreadDeath) { - logger.info("JOGL threw ThreadDeath") + + runtime += 200.milliseconds + delay(200) + } + + if(failure) { + exampleRunnable.cancelAndJoin() + } else { + exampleRunnable.join() } logger.info("${clazz.simpleName} closed.") @@ -145,7 +143,7 @@ class ExampleRunner( // find all basic and advanced examples, exclude blacklist val examples = ClassGraph() - .acceptPackages("graphics.scenery.tests") + .acceptPackages("graphics.scenery.tests.examples") .enableClassInfo() .scan() .getSubclasses(SceneryBase::class.java) diff --git a/src/test/kotlin/graphics/scenery/tests/examples/volumes/BDVExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/volumes/BDVExample.kt index d274ea722e..56a8df7a9f 100644 --- a/src/test/kotlin/graphics/scenery/tests/examples/volumes/BDVExample.kt +++ b/src/test/kotlin/graphics/scenery/tests/examples/volumes/BDVExample.kt @@ -86,7 +86,7 @@ class BDVExample: SceneryBase("BDV Rendering example", 1280, 720) { v.viewerState.sources.firstOrNull()?.spimSource?.getSource(0, 0)?.let { rai -> var h: Any? val duration = measureTimeMillis { - h = ops.run("image.histogram", rai, 1024) + h = ops.run("image.histogram", rai, 32) } val histogram = h as? Histogram1d<*> ?: return@let diff --git a/src/test/kotlin/graphics/scenery/tests/examples/volumes/ProceduralVolumeExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/volumes/ProceduralVolumeExample.kt index fbf7ba9516..8e06dc5e21 100644 --- a/src/test/kotlin/graphics/scenery/tests/examples/volumes/ProceduralVolumeExample.kt +++ b/src/test/kotlin/graphics/scenery/tests/examples/volumes/ProceduralVolumeExample.kt @@ -5,6 +5,7 @@ import graphics.scenery.* import graphics.scenery.backends.Renderer import graphics.scenery.numerics.Random import graphics.scenery.attribute.material.Material +import graphics.scenery.controls.behaviours.ArcballCameraControl import graphics.scenery.utils.RingBuffer import graphics.scenery.utils.extensions.plus import graphics.scenery.volumes.BufferedVolume @@ -124,7 +125,10 @@ class ProceduralVolumeExample: SceneryBase("Procedural Volume Rendering Example" /** * Input setup override, sets up camera mode switching, where pressing C * can toggle between FPS and Arcball camera control. Also adds animation - * toggling when pressing T. + * toggling when pressing T. Additionally, demonstrates how the + * rotateDegrees of ArcballCameraControl can be used to rotate the camera + * by a fixed amount (here, 10 degrees yaw) about the scene origin, by + * pressing R. */ override fun inputSetup() { setupCameraModeSwitching() @@ -134,6 +138,14 @@ class ProceduralVolumeExample: SceneryBase("Procedural Volume Rendering Example" volume.metadata["animating"] = !(volume.metadata["animating"] as Boolean) }) inputHandler?.addKeyBinding("toggle_animation", "T") + + val arcballCameraControl = ArcballCameraControl("fixed_rotation", { scene.findObserver()!! }, windowWidth, windowHeight, scene.findObserver()!!.target) + inputHandler?.addBehaviour("rotate_camera", + ClickBehaviour { _, _ -> + arcballCameraControl.rotateDegrees(10f, 0f) + }) + inputHandler?.addKeyBinding("rotate_camera", "R") + } /** diff --git a/src/test/kotlin/graphics/scenery/tests/examples/volumes/SplitVolumeExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/volumes/SplitVolumeExample.kt new file mode 100644 index 0000000000..f250156e8e --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/volumes/SplitVolumeExample.kt @@ -0,0 +1,93 @@ +package graphics.scenery.tests.examples.volumes + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.controls.TrackedStereoGlasses +import graphics.scenery.attribute.material.Material +import graphics.scenery.utils.extensions.positionVolumeSlices +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import net.imglib2.type.numeric.integer.UnsignedShortType +import org.joml.Vector3f +import java.nio.file.Paths + +/** + * Volume rendering on a volume file partitioned into sliced buffers before loading into the scene. + * + * @author Aryaman Gupta + */ +class SplitVolumeExample: SceneryBase("Split volume data", 1280, 720) { + var hmd: TrackedStereoGlasses? = null + + override fun init() { + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + val cam: Camera = DetachedHeadCamera(hmd) + with(cam) { + spatial { + position = Vector3f(0.0f, 0.5f, 5.0f) + } + perspectiveCamera(50.0f, windowWidth, windowHeight) + + scene.addChild(this) + } + + val shell = Box(Vector3f(10.0f, 10.0f, 10.0f), insideNormals = true) + shell.material { + cullingMode = Material.CullingMode.None + diffuse = Vector3f(0.2f, 0.2f, 0.2f) + specular = Vector3f(0.0f) + ambient = Vector3f(0.0f) + } + scene.addChild(shell) + + val s = Icosphere(0.5f, 3) + s.spatial().position = Vector3f(2.0f, -1.0f, -2.0f) + s.material().diffuse = Vector3f(0.0f, 0.0f, 0.0f) + scene.addChild(s) + + val pair = Volume.fromPathRawSplit(Paths.get(getDemoFilesPath() + "/volumes/box-iso/box_200_200_200.raw"), hub = hub, type = UnsignedShortType(), sizeLimit = 20000000) + val parent = pair.first as RichNode + val volumeList = pair.second + + parent.positionVolumeSlices(volumeList) + + volumeList.forEachIndexed{ i, volume-> + volume.name = "volume_$i" + volume.colormap = Colormap.get("viridis") + volume.transferFunction = TransferFunction.ramp(0.1f, 0.5f) + volume.origin = Origin.Center + volume.spatial().scale = Vector3f(20.0f, 20.0f, 20.0f) + } + + parent.spatial { + position = Vector3f(0.0f, 0.0f, -3.5f) + rotation = rotation.rotateXYZ(0.05f, 0.05f, 0.05f) + } + + scene.addChild(parent) + + val lights = (0 until 3).map { + PointLight(radius = 15.0f) + } + + lights.mapIndexed { i, light -> + light.spatial().position = Vector3f(2.0f * i - 4.0f, i - 1.0f, 0.0f) + light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + light.intensity = 0.5f + scene.addChild(light) + } + } + + override fun inputSetup() { + setupCameraModeSwitching() + } + + companion object { + @JvmStatic + fun main(args: Array) { + SplitVolumeExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/volumes/VDIGenerationExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/volumes/VDIGenerationExample.kt new file mode 100644 index 0000000000..c3b11458a4 --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/volumes/VDIGenerationExample.kt @@ -0,0 +1,173 @@ +package graphics.scenery.tests.examples.volumes + +import graphics.scenery.* +import graphics.scenery.volumes.VolumeManager +import graphics.scenery.backends.Renderer +import graphics.scenery.volumes.Colormap +import graphics.scenery.volumes.TransferFunction +import graphics.scenery.volumes.Volume +import graphics.scenery.volumes.vdi.* +import org.joml.Vector3f +import java.nio.file.Paths +import java.util.concurrent.atomic.AtomicInteger +import graphics.scenery.backends.vulkan.VulkanRenderer +import graphics.scenery.utils.SystemHelpers +import graphics.scenery.volumes.* +import org.joml.* +import java.io.* +import java.nio.ByteBuffer +import kotlin.concurrent.thread + +/** + * Example application showing how to generate Volumetric Depth Images (VDIs). A [VolumeManager] is setup to generate + * VDIs and the generated VDIs are written to the file system. + * + * @author Aryaman Gupta + */ +class VDIGenerationExample(wWidth: Int = 512, wHeight: Int = 512, val maxSupersegments: Int = 20) : SceneryBase("Volume Generation Example", wWidth, wHeight) { + private var count = 0 + + override fun init() { + + // Step 1: Create renderer, volume and camera + renderer = hub.add( + SceneryElement.Renderer, + Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + val cam: Camera = DetachedHeadCamera() + with(cam) { + spatial { + position = Vector3f(0.0f, 0.5f, 5.0f) + } + perspectiveCamera(50.0f, windowWidth, windowHeight) + scene.addChild(this) + } + + val volume = Volume.fromPathRaw(Paths.get(getDemoFilesPath() + "/volumes/box-iso/"), hub) + volume.name = "volume" + volume.colormap = Colormap.get("viridis") + volume.spatial { + position = Vector3f(0.0f, 0.0f, -3.5f) + rotation = rotation.rotateXYZ(0.05f, 0.05f, 0.05f) + scale = Vector3f(20.0f, 20.0f, 20.0f) + } + volume.transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(volume) + + // Step 2: Create VDI Volume Manager + val vdiVolumeManager = VDIVolumeManager( hub, windowWidth, windowHeight, maxSupersegments, scene).createVDIVolumeManager() + + //step 3: switch the volume's current volume manager to VDI volume manager + volume.volumeManager = vdiVolumeManager + + // Step 4: add the volume to VDI volume manager + vdiVolumeManager.add(volume) + volume.volumeManager.shaderProperties["doGeneration"] = true + + // Step 5: add the VDI volume manager to the hub + hub.add(vdiVolumeManager) + + // Step 6: Store VDI Generated + val volumeDimensions3i = Vector3f(volume.getDimensions().x.toFloat(),volume.getDimensions().y.toFloat(),volume.getDimensions().z.toFloat()) + val model = volume.spatial().world + + val vdiData = VDIData( + VDIBufferSizes(), + VDIMetadata( + index = count, + projection = cam.spatial().projection, + view = cam.spatial().getTransformation(), + volumeDimensions = volumeDimensions3i, + model = model, + nw = volume.volumeManager.shaderProperties["nw"] as Float, + windowDimensions = Vector2i(cam.width, cam.height) + ) + ) + + thread(isDaemon = true) { + storeVDI(vdiVolumeManager, vdiData) + } + } + + private fun storeVDI(vdiVolumeManager: VolumeManager, vdiData: VDIData) { + data class Timer(var start: Long, var end: Long) + val tGeneration = Timer(0, 0) + + var vdiDepthBuffer: ByteBuffer? + var vdiColorBuffer: ByteBuffer? + var gridCellsBuff: ByteBuffer? + + val volumeList = ArrayList() + volumeList.add(vdiVolumeManager.nodes.first() as BufferedVolume) + val vdisGenerated = AtomicInteger(0) + while (renderer?.firstImageReady == false) { + Thread.sleep(50) + } + + val vdiColor = vdiVolumeManager.material().textures[VDIVolumeManager.colorTextureName]!! + val colorCnt = AtomicInteger(0) + (renderer as? VulkanRenderer)?.persistentTextureRequests?.add(vdiColor to colorCnt) + + val vdiDepth = vdiVolumeManager.material().textures[VDIVolumeManager.depthTextureName]!! + val depthCnt = AtomicInteger(0) + (renderer as? VulkanRenderer)?.persistentTextureRequests?.add(vdiDepth to depthCnt) + + + val gridCells = vdiVolumeManager.material().textures[VDIVolumeManager.accelerationTextureName]!! + val gridTexturesCnt = AtomicInteger(0) + (renderer as? VulkanRenderer)?.persistentTextureRequests?.add(gridCells to gridTexturesCnt) + + var prevColor = colorCnt.get() + var prevDepth = depthCnt.get() + + // TODO: convert VDI storage also to postRenderLambda + while (count < 6) { + + tGeneration.start = System.nanoTime() + + while (colorCnt.get() == prevColor || depthCnt.get() == prevDepth) { + Thread.sleep(5) + } + + prevColor = colorCnt.get() + prevDepth = depthCnt.get() + + vdiColorBuffer = vdiColor.contents + vdiDepthBuffer = vdiDepth.contents + gridCellsBuff = gridCells.contents + + tGeneration.end = System.nanoTime() + + val timeTaken = (tGeneration.end - tGeneration.start) / 1e9 + + logger.info("Time taken for generation (only correct if VDIs were not being written to disk): ${timeTaken}") + + vdiData.metadata.index = count + + if (count == 4) { //store the 4th VDI + + val vdiFilename = "example$count" + val file = FileOutputStream(File("$vdiFilename.vdi-metadata")) + VDIDataIO.write(vdiData, file) + logger.info("written the dump") + file.close() + + SystemHelpers.dumpToFile(vdiColorBuffer!!, "$vdiFilename.vdi-color") + SystemHelpers.dumpToFile(vdiDepthBuffer!!, "$vdiFilename.vdi-depth") + SystemHelpers.dumpToFile(gridCellsBuff!!, "$vdiFilename.vdi-grid") + + logger.info("Wrote VDI $count") + vdisGenerated.incrementAndGet() + } + count++ + } + } + + + companion object { + @JvmStatic + fun main(args: Array) { + VDIGenerationExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/volumes/VDIRenderingExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/volumes/VDIRenderingExample.kt new file mode 100644 index 0000000000..d6d45ec09e --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/volumes/VDIRenderingExample.kt @@ -0,0 +1,109 @@ +package graphics.scenery.tests.examples.volumes + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.volumes.vdi.VDIDataIO +import graphics.scenery.volumes.vdi.VDINode +import org.joml.Vector3f +import org.lwjgl.system.MemoryUtil +import java.io.File +import java.io.FileInputStream +import java.io.FileNotFoundException +import java.nio.ByteBuffer + +/** + * Example showing how a VDI can be rendered. + * + * @author Aryaman Gupta + */ +class VDIRenderingExample : SceneryBase("VDI Rendering Example", 512, 512) { + + val vdiFilename = "example4" + val skipEmpty = false + + val numSupersegments = 20 + + lateinit var vdiNode: VDINode + val numLayers = 1 + + val cam: Camera = DetachedHeadCamera() + + override fun init() { + + // Step 1: create a Renderer, Point light and camera + renderer = hub.add(Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + val light = PointLight(radius = 15.0f) + light.spatial().position = Vector3f(0.0f, 0.0f, 2.0f) + light.intensity = 5.0f + light.emissionColor = Vector3f(1.0f, 1.0f, 1.0f) + scene.addChild(light) + + with(cam) { + spatial().position = Vector3f(0.0f, 0.5f, 5.0f) + perspectiveCamera(50.0f, windowWidth, windowWidth) + scene.addChild(this) + } + + // Step 2: read files + val file = try { + FileInputStream(File("$vdiFilename.vdi-metadata")) + } catch(e: FileNotFoundException) { + logger.error("File ${vdiFilename}.vdi-metadata not found!") + return + } + + val vdiData = VDIDataIO.read(file) + logger.info("Fetching file...") + + vdiNode = VDINode(windowWidth, windowHeight, numSupersegments, vdiData) + + val colorArray: ByteArray = File("$vdiFilename.vdi-color").readBytes() + val depthArray: ByteArray = File("$vdiFilename.vdi-depth").readBytes() + val octArray: ByteArray = File("$vdiFilename.vdi-grid").readBytes() + + // Step 3: assigning buffer values + val colBuffer: ByteBuffer = MemoryUtil.memCalloc(vdiNode.vdiHeight * vdiNode.vdiWidth * numSupersegments * numLayers * 4 * 4) + colBuffer.put(colorArray).flip() + colBuffer.limit(colBuffer.capacity()) + + val depthBuffer = MemoryUtil.memCalloc(vdiNode.vdiHeight * vdiNode.vdiWidth * numSupersegments * 2 * 2 * 2) + depthBuffer.put(depthArray).flip() + depthBuffer.limit(depthBuffer.capacity()) + + val numGridCells = vdiNode.getAccelerationGridSize() + + val gridBuffer = MemoryUtil.memAlloc(numGridCells.x.toInt() * numGridCells.y.toInt() * numGridCells.z.toInt() * 4) + if(skipEmpty) { + gridBuffer.put(octArray).flip() + gridBuffer.limit(gridBuffer.capacity()) + } + + //Step 4: Attaching the buffers to the vdi node and adding it to the scene + vdiNode.attachTextures(colBuffer, depthBuffer, gridBuffer) + + vdiNode.skip_empty = skipEmpty + + //Attaching empty textures as placeholders for 2nd VDI buffer, which is unused here + vdiNode.attachEmptyTextures(VDINode.DoubleBuffer.Second) + + scene.addChild(vdiNode) + + val plane = FullscreenObject() + plane.material().textures["diffuse"] = vdiNode.material().textures["OutputViewport"]!! + scene.addChild(plane) + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + VDIRenderingExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/examples/volumes/VolumeManagerSwitchingExample.kt b/src/test/kotlin/graphics/scenery/tests/examples/volumes/VolumeManagerSwitchingExample.kt new file mode 100644 index 0000000000..9ed8250f8f --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/examples/volumes/VolumeManagerSwitchingExample.kt @@ -0,0 +1,78 @@ +package graphics.scenery.tests.examples.volumes + +import graphics.scenery.* +import graphics.scenery.backends.Renderer +import graphics.scenery.volumes.* +import graphics.scenery.volumes.vdi.VDIVolumeManager +import org.joml.Vector3f +import java.nio.file.Paths +import kotlin.concurrent.thread + +/** + * The `VolumeManagerSwitchingExample` class demonstrates how to switch between two different volume managers. + * + * It creates a VDI volume manager for the volume and switches between the VDI volume manager and the standard + * volume manager at regular intervals using the replace function. + * + */ +class VolumeManagerSwitchingExample : SceneryBase("Volume Manager Switching Example", 512, 512) { + + val maxSupersegments = System.getProperty("VolumeBenchmark.NumSupersegments")?.toInt()?: 20 + override fun init() { + + //Step 1: First create a volume, camera , renderer ... + renderer = hub.add( + SceneryElement.Renderer, + Renderer.createRenderer(hub, applicationName, scene, windowWidth, windowHeight)) + + val cam: Camera = DetachedHeadCamera() + with(cam) { + spatial { + position = Vector3f(0.0f, 0.5f, 5.0f) + } + perspectiveCamera(50.0f, windowWidth, windowHeight) + scene.addChild(this) + } + + val volume = Volume.fromPathRaw(Paths.get(getDemoFilesPath() + "/volumes/box-iso/"), hub) + volume.name = "volume" + volume.colormap = Colormap.get("viridis") + volume.spatial { + position = Vector3f(0.0f, 0.0f, -3.5f) + rotation = rotation.rotateXYZ(0.05f, 0.05f, 0.05f) + scale = Vector3f(20.0f, 20.0f, 20.0f) + } + volume.transferFunction = TransferFunction.ramp(0.1f, 0.5f) + scene.addChild(volume) + + //Step 2: create a volume manager for vdi and add the volume to it: + val vdiVolumeManager = VDIVolumeManager( hub, windowWidth, windowHeight, maxSupersegments, scene).createVDIVolumeManager() + vdiVolumeManager.add(volume) + + //Step 3: save the standard volume manger, the one who was first created with the volume + val standardVolumeManager : VolumeManager = hub.get() as VolumeManager + + //Step 4: switch between different volume managers + thread { + while (true) { + Thread.sleep(4000) + vdiVolumeManager.replace(vdiVolumeManager) + Thread.sleep(4000) + standardVolumeManager.replace(standardVolumeManager) + } + } + } + + /** + * Companion object for providing a main method. + */ + companion object { + /** + * The main entry point. Executes this example application when it is called. + */ + @JvmStatic + fun main(args: Array) { + VolumeManagerSwitchingExample().main() + } + } +} diff --git a/src/test/kotlin/graphics/scenery/tests/unit/utils/DataCompressorTests.kt b/src/test/kotlin/graphics/scenery/tests/unit/utils/DataCompressorTests.kt new file mode 100644 index 0000000000..21ab2e922d --- /dev/null +++ b/src/test/kotlin/graphics/scenery/tests/unit/utils/DataCompressorTests.kt @@ -0,0 +1,62 @@ +package graphics.scenery.tests.unit.utils + +import graphics.scenery.utils.DataCompressor +import graphics.scenery.utils.lazyLogger +import org.junit.Test +import org.lwjgl.system.MemoryUtil +import java.util.* +import kotlin.test.assertTrue + +/** + * Tests how [DataCompressor] can be used for lossless compression and decompression of binary data. + * + * @author Aryaman Gupta + */ +class DataCompressorTests { + private val logger by lazyLogger() + + /** + * Tests compression and decompression of data. + */ + @Test + fun testCompressionDecompression() { + val dataSize = 1024*1024*8 + val buffer = MemoryUtil.memAlloc(dataSize) + + // insert random integers between 0 and 5 into the buffer + val rd = Random() + val intBuffer = buffer.asIntBuffer() + for(i in 0 until intBuffer.remaining()) { + intBuffer.put(rd.nextInt(5)) + } + + val compressionTool = DataCompressor.CompressionTool.LZ4 + val compressor = DataCompressor(compressionTool) + + val maxDecompressedSize = compressor.returnCompressBound(buffer.remaining().toLong()) + val compressedBuffer = MemoryUtil.memAlloc(maxDecompressedSize) + + val compressedLength = compressor.compress(compressedBuffer, buffer, 3) + compressedBuffer.limit(compressedLength.toInt()) + + val compressionRatio = compressedLength.toFloat() / dataSize.toFloat() + logger.info("Length of compressed buffer: $compressedLength and compression ration is: $compressionRatio") + + val decompressed = MemoryUtil.memAlloc(dataSize) + compressor.decompress(decompressed, compressedBuffer) + + val successful = compressor.verifyDecompressed(buffer, decompressed) + if(successful) { + logger.info("The buffer was found to be compressed and decompressed successfully") + } else { + logger.info("Compression and decompression was not successful") + } + + assertTrue { + compressor.verifyDecompressed(buffer, decompressed) + } + + MemoryUtil.memFree(buffer) + MemoryUtil.memFree(decompressed) + } +} \ No newline at end of file diff --git a/src/test/resources/graphics/scenery/tests/examples/advanced/CheckDataForAsyncExample.comp b/src/test/resources/graphics/scenery/tests/examples/advanced/CheckDataForAsyncExample.comp new file mode 100644 index 0000000000..cdaa3919d0 --- /dev/null +++ b/src/test/resources/graphics/scenery/tests/examples/advanced/CheckDataForAsyncExample.comp @@ -0,0 +1,14 @@ +#version 450 + +layout (local_size_x = 16, local_size_y = 16) in; +layout (set = 0, binding = 0, r8) uniform readonly image3D humongous; +layout (set = 1, binding = 0, rgba8) uniform image2D OutputViewport; + +void main() { + + int depth = imageSize(humongous).z; + + vec4 color = imageLoad(humongous, ivec3(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y, depth/2)); + + imageStore(OutputViewport, ivec2(gl_GlobalInvocationID.x, gl_GlobalInvocationID.y), vec4(color.rrr, 1.0)); +} diff --git a/src/test/resources/graphics/scenery/tests/examples/compute/ComputeVolume.comp b/src/test/resources/graphics/scenery/tests/examples/compute/ComputeVolume.comp index 42e0bb398f..43a949cc49 100644 --- a/src/test/resources/graphics/scenery/tests/examples/compute/ComputeVolume.comp +++ b/src/test/resources/graphics/scenery/tests/examples/compute/ComputeVolume.comp @@ -59,6 +59,13 @@ void intersectBox( vec3 r_o, vec3 r_d, vec3 boxmin, vec3 boxmax, out float tnear tfar = min( min( tmax.x, tmax.y ), min( tmax.x, tmax.z ) ); } +float adjustOpacity(float a, float modifiedStepLength) { + return 1.0 - pow((1.0 - a), modifiedStepLength); +} + +uniform bool fixedStepSize; +uniform float stepsPerVoxel; + // --------------------- // $insert{Convert} // $insert{SampleVolume} @@ -93,12 +100,16 @@ void main() float tnear = 1, tfar = 0, tmax = getMaxDepth( depthUV ); float n, f; - // $repeat:{vis,intersectBoundingBox| + // $repeat:{vis,localNear,localFar,intersectBoundingBox| bool vis = false; + float localNear = 0.0f; + float localFar = 0.0f; intersectBoundingBox( wfront, wback, n, f ); f = min( tmax, f ); if ( n < f ) { + localNear = n; + localFar = f; tnear = min( tnear, max( 0, n ) ); tfar = max( tfar, f ); vis = true; @@ -115,15 +126,28 @@ void main() ? int ( log( ( tfar * fwnw + nw ) / ( tnear * fwnw + nw ) ) / log ( 1 + fwnw ) ) : int ( trunc( ( tfar - tnear ) / nw + 1 ) ); + float stepWidth = nw; + + if(fixedStepSize) { + stepWidth = (2*nw) / stepsPerVoxel; + numSteps = int ( trunc( ( tfar - tnear ) / stepWidth + 1 ) ); + } + float step = tnear; + vec4 w_entry = mix(wfront, wback, step); + + float standardStepSize = distance(mix(wfront, wback, step + nw), w_entry); + + float step_prev = step - stepWidth; + vec4 wprev = mix(wfront, wback, step_prev); vec4 v = vec4( 0 ); - for ( int i = 0; i < numSteps; ++i, step += nw + step * fwnw ) + for ( int i = 0; i < numSteps; ++i) { vec4 wpos = mix( wfront, wback, step ); // $insert{Accumulate} /* - inserts something like the following (keys: vis,blockTexture,convert) + inserts something like the following (keys: vis,localNear,localFar,blockTexture,convert) if (vis) { @@ -131,6 +155,13 @@ void main() v = max(v, convert(x)); } */ + wprev = wpos; + + if(fixedStepSize) { + step += stepWidth; + } else { + step += nw + step * fwnw; + } } FragColor = v; } else {