Skip to content

Commit

Permalink
First draw of a Benchmarking framework
Browse files Browse the repository at this point in the history
  • Loading branch information
marius-pelegrin-arm committed Sep 19, 2024
1 parent 681a117 commit 244b62d
Show file tree
Hide file tree
Showing 12 changed files with 1,190 additions and 90 deletions.
9 changes: 5 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
cmake_minimum_required(VERSION 3.13)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED YES)
enable_testing()
project(tracetooltests)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED YES)

if(NOT DEFINED WINDOWSYSTEM) # to prevent overwriting toolchain setting
set(WINDOWSYSTEM "x11" CACHE STRING "Window system to use for build")
Expand Down Expand Up @@ -48,7 +47,8 @@ set(SYMLINK_DIR "$ENV{HOME}/.local/share/benchmarking")
install(DIRECTORY DESTINATION "${SYMLINK_DIR}")

function(gles_test test_name)
add_executable(gles_${ARGV0} src/util.cpp src/util.h src/gles_${ARGV0}.cpp src/gles_common.cpp src/gles_common.h)
add_executable(gles_${ARGV0} src/util.cpp src/util.h src/gles_${ARGV0}.cpp src/gles_common.cpp src/gles_common.h
src/benchmarking/generated_benchmarking.hpp src/benchmarking/generated_benchmarking.cpp src/benchmarking/benchmarking.hpp src/benchmarking/benchmarking.cpp)
target_link_libraries(gles_${ARGV0} PRIVATE -Wl,--add-needed EGL GLESv2 pthread ${IT_LIBS})
target_link_directories(gles_${ARGV0} PRIVATE ${API_LIBS})
target_compile_definitions(gles_${ARGV0} PUBLIC ${IT_DEFINES})
Expand All @@ -63,7 +63,8 @@ endfunction()

function(vulkan_test_build test_name)
add_executable(vulkan_${ARGV0} src/vulkan_${ARGV0}.cpp src/vulkan_common.cpp src/vulkan_common.h src/vulkan_compute_common.cpp src/vulkan_compute_common.h
src/vulkan_window_common.cpp src/vulkan_window_common.h src/util.cpp src/util.h)
src/vulkan_window_common.cpp src/vulkan_window_common.h src/util.cpp src/util.h
src/benchmarking/generated_benchmarking.hpp src/benchmarking/generated_benchmarking.cpp src/benchmarking/benchmarking.hpp src/benchmarking/benchmarking.cpp)
target_link_libraries(vulkan_${ARGV0} PRIVATE vulkan pthread ${IT_LIBS} ${XCB_LIBRARIES})
target_link_directories(vulkan_${ARGV0} PRIVATE ${API_LIBS})
target_compile_definitions(vulkan_${ARGV0} PUBLIC ${IT_DEFINES})
Expand Down
56 changes: 37 additions & 19 deletions doc/BenchmarkingStandard.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,24 @@ Here's an example of capabilities JSON file:
"setting_1_name": {
"description": "Human readable description of setting 1",
"type": "selection",
"default": "high quality",
"options": [ "high performance", "balanced", "high quality" ]
},
"setting_2_name": {
"description": "Human readable description of setting 2",
"type": "bool"
"type": "bool",
"default": true
},
"setting_3_name": {
"description": "Human readable description of setting 3",
"type": "number",
"type": "integer",
"default": 27,
"min": 0,
"max": 100
},
"setting_4_name": {
"description": "Human readable description of setting 4",
"type": "float",
"min": 0,
"max": 3.14
}
Expand All @@ -69,8 +78,12 @@ Here's an example of capabilities JSON file:
}
},
"adaptations": {
"adaptation_1_name": "Human readable description of adaptation 1",
"adaptation_2_name": "Human readable description of adaptation 2"
"adaptation_1_name": {
"description": "Human readable description of adaptation 1"
},
"adaptation_2_name": {
"description": "Human readable description of adaptation 2"
}
}
}
```
Expand Down Expand Up @@ -98,7 +111,6 @@ Available fields that can be added in each scene entry:

| Field | Mandatory | Description |
| ----- | --------- | ----------- |
| name | Yes | A name for the scene that can later be used to activate it |
| description | No | A human readable description of the scene |

#### Settings
Expand All @@ -107,11 +119,12 @@ Available fields that can be added in each settings entry:

| Field | Mandatory | Description |
| ----- | --------- | ----------- |
| description | Yes | A short human readable description of the option |
| type | Yes | One of `selection`, `bool`, or `number` |
| options | If type is `selection` | If type is `selection`, this must be an array of possible values for the setting |
| min | If type is `number` | If type is `number`, this must be the minimum value for this setting |
| max | If type is `number` | If type is `number`, this must be the maximum value for this setting |
| description | No | A short human readable description of the option |
| type | Yes | One of `selection`, `bool`, `integer` or `float` |
| default | No | Default value if the setting value is not specified in the enable file |
| options | If type is `selection` | Array of possible values for the setting |
| min | If type is `integer` or `float` | Minimum value for this setting |
| max | If type is `integer` or `float` | Maximum value for this setting |

#### Capabilities

Expand All @@ -126,11 +139,11 @@ The capabilities entry is a list of key-value pairs, where the key is the name o
| disable_os_adaptations | bool | {true, false} | As above, but for adaptations that have been added to prevent rendering artifacts or crashes on specific versions of an operating or windowing system version system. |
| disable_loading_screen | bool | {true, false} | The application is capable of turning off any loading screen that is shown while the application is starting up or changing scenes. Instead, no new frames should be rendered. |
| non_interactive | bool | {true, false} | The application is capable of running in fully non-interactive mode. If enabled, it must automatically run the activated scenes (or some default scene if none given or scene selection not supported), and then if able to run it to completion, exit with a successful exit code. If it fails to complete the scene, it shall exit with a failure exit code. |
| fixed_framerate | number | [0, inf] | The application is capable of running in a fixed framerate mode. This means that no matter the performance it will render the same content in the same number of frames every time. If the enable value is set to a non-zero value, this is the number of desired milliseconds of simulated rendering time between each frame. If zero, an application provided default frame time shall be used. |
| visual_settings | number | [1, 100] | The application is capable of tuning its rendering quality by giving it a 1-100 value, where 1 is lowest and 100 is highest. |
| loops | number | [0, inf] | The application is capable of running its scenes in a loop with no or only a minimal scene loading between each loop iteration. The enable value gives the number of loops to be run, where `0` means run in an infinite loop. This is useful in particular for measuring sustained performance, or measuring power, battery and temperature. If multiple scenes are activated, the order in which they loop is up to the application. |
| loop_time | number | [0, inf] | The application is capable of running its scenes in a loop as above but for a certain amount of time. If present in the enable file, this gives the number of seconds to run the loop for. |
| gpu_delay_reuse | number | [1, inf] | The application is capable of delaying the reuse of GPU resources. The enable value gives the number of frames that resources must not be reused for. |
| fixed_framerate | float | [0, inf] | The application is capable of running in a fixed framerate mode. This means that no matter the performance it will render the same content in the same number of frames every time. If the enable value is set to a non-zero value, this is the number of desired milliseconds of simulated rendering time between each frame. If zero, an application provided default frame time shall be used. |
| visual_settings | float | [1, 100] | The application is capable of tuning its rendering quality by giving it a 1-100 value, where 1 is lowest and 100 is highest. |
| loops | int | [0, inf] | The application is capable of running its scenes in a loop with no or only a minimal scene loading between each loop iteration. The enable value gives the number of loops to be run, where `0` means run in an infinite loop. This is useful in particular for measuring sustained performance, or measuring power, battery and temperature. If multiple scenes are activated, the order in which they loop is up to the application. |
| loop_time | float | [0, inf] | The application is capable of running its scenes in a loop as above but for a certain amount of time. If present in the enable file, this gives the number of seconds to run the loop for. |
| gpu_delay_reuse | int | [1, inf] | The application is capable of delaying the reuse of GPU resources. The enable value gives the number of frames that resources must not be reused for. |
| gpu_no_coherent | bool | {true, false} | The application is capable of behaving as if no coherent memory exists, and must explicitly call the graphics API to flush any modified memory before it is to be used for rendering. This allows for instance gfxreconstruct to run in 'assisted' tracing mode instead of using guard pages which may speed up runtime performance while tracing or avoid issues with guard pages. |
| gpu_frame_deterministic | bool | {true, false} | Whether the application generates a deterministic final rendering output. |
| gpu_fully_deterministic | bool | {true, false} | Whether the application generates deterministic rendering outputs from every GPU compute or rendering step. |
Expand All @@ -146,10 +159,15 @@ Fields that can be added in each capability properties entry:

#### Adaptations

The adaptations entry is a list of key-value pairs, where the key is an application-defined name, and the value is a free-form description of what it does.

An adaptation is a behaviour that may be enabled by the application depending on runtime heuristics. The user may use the enable file to force this adaptation on or off, bypassing the runtime heuristics.

Available fields that can be added in each adaptation entry:

| Field | Mandatory | Description |
| ----- | --------- | ----------- |
| name | Yes | A name for the adaptation that can later be used to activate it |
| description | No | A human readable description of the adaptation |

## File locations

Describe where the capability file must be located for each platform. It needs to be predictable so that third-party tools can locate them automatically.
Expand Down Expand Up @@ -203,12 +221,12 @@ Here's an example of enable file associated with the example of capabilities fil
{
"target": "application.name",
"scenes": [ "scene_1_name", "scene_2_name" ],
"results": "path/to/results/file",
"intent": "showcase",
"results": "path/to/results/file",
"settings": {
"setting_1_name": "balanced",
"setting_2_name": false,
"setting_3_name": 1.618
"setting_4_name": 1.618
},
"capabilities": {
"fixed_framerate": 60
Expand Down
78 changes: 78 additions & 0 deletions doc/benchmarking_registry.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
{
"capabilities": {
"disable_cpu_performance_adaptations": {
"type": "bool",
"description": "The application is capable of disabling dynamic runtime CPU adaptations to improve performance. This also include using settings that are adapted to application prior knowledge of the device performance and core pinning, but not sizing thread pools according to CPU core numbers."
},
"disable_gpu_performance_adaptations": {
"type": "bool",
"description": "The application is capable of disabling dynamic runtime graphics adaptations to improve performance. This also include using settings that are adapted to application prior knowledge of the device performance."
},
"disable_vendor_performance_adaptations": {
"type": "bool",
"description": "The application is capable of disabling vendor specific adaptations to improve performance. If specific adaptations are exposed in the capabilities file and are enabled in the enable file, those adaptations should override changes made by this option."
},
"disable_vendor_adaptations": {
"type": "bool",
"description": "The application is capable of disabling vendor specific bug adaptations that have been added to prevent rendering artifacts or crashes. If specific adaptations are exposed in the capabilities file and are enabled in the enable file, those adaptations should override changes made by this option."
},
"disable_os_adaptations": {
"type": "bool",
"description": "As above, but for adaptations that have been added to prevent rendering artifacts or crashes on specific versions of an operating or windowing system version system."
},
"disable_loading_screen": {
"type": "bool",
"description": "The application is capable of turning off any loading screen that is shown while the application is starting up or changing scenes. Instead, no new frames should be rendered."
},
"non_interactive": {
"type": "bool",
"description": "The application is capable of running in fully non-interactive mode. If enabled, it must automatically run the activated scenes (or some default scene if none given or scene selection not supported), and then if able to run it to completion, exit with a successful exit code. If it fails to complete the scene, it shall exit with a failure exit code."
},
"fixed_framerate": {
"type": "float",
"min": 0,
"description": "The application is capable of running in a fixed framerate mode. This means that no matter the performance it will render the same content in the same number of frames every time. If the enable value is set to a non-zero value, this is the number of desired milliseconds of simulated rendering time between each frame. If zero, an application provided default frame time shall be used."
},
"visual_settings": {
"type": "float",
"min": 1,
"max": 100,
"description": "The application is capable of tuning its rendering quality by giving it a 1-100 value, where 1 is lowest and 100 is highest."
},
"loops": {
"type": "int",
"min": 0,
"description": "The application is capable of running its scenes in a loop with no or only a minimal scene loading between each loop iteration. The enable value gives the number of loops to be run, where 0 means run in an infinite loop. This is useful in particular for measuring sustained performance, or measuring power, battery and temperature. If multiple scenes are activated, the order in which they loop is up to the application."
},
"loop_time": {
"type": "float",
"min": 0,
"description": "The application is capable of running its scenes in a loop as above but for a certain amount of time. If present in the enable file, this gives the number of seconds to run the loop for."
},
"gpu_delay_reuse": {
"type": "int",
"min": 1,
"description": "The application is capable of delaying the reuse of GPU resources. The enable value gives the number of frames that resources must not be reused for."
},
"gpu_no_coherent": {
"type": "bool",
"description": "The application is capable of behaving as if no coherent memory exists, and must explicitly call the graphics API to flush any modified memory before it is to be used for rendering. This allows for instance gfxreconstruct to run in 'assisted' tracing mode instead of using guard pages which may speed up runtime performance while tracing or avoid issues with guard pages."
},
"gpu_frame_deterministic": {
"type": "bool",
"description": "Whether the application generates a deterministic final rendering output."
},
"gpu_fully_deterministic": {
"type": "bool",
"description": "Whether the application generates deterministic rendering outputs from every GPU compute or rendering step."
},
"frameless": {
"type": "bool",
"description": "Whether the application does not present graphical frames. If opt-out and frameless is disabled in the enable file, then it must add present or frame boundary calls."
},
"file_output": {
"type": "bool",
"description": "Whether the application is capable of or will generate outputs for each scene as files in the filesystem."
}
}
}
Loading

0 comments on commit 244b62d

Please sign in to comment.