From 0c3a994b90d4217fcbdc1078fdbddf4091dbb888 Mon Sep 17 00:00:00 2001 From: jakob Date: Sat, 29 Jun 2024 02:00:31 +0200 Subject: [PATCH] [build] Check disable-nxcompat flag for standalone A Win32 executable binary needs to disable NXCOMPAT mode to allow AVS to execute code from writeable memory. Marking a binary with NXCOMPAT disallows write-exec permissions for it. By default (at least in newer binutils releases) binaries will automatically be marked as such. Alternatively the linker may mark binaries with "NXCOMPAT:NO", meaning they do not support the write-exec prevention. This is meant for compiling legacy applications (such as AVS). Binutils 2.36 is the oldest version that supports --disable-nxcompat. MSVC should generally support /NXCOMPAT:NO. If the linker does not support the flag, don't add the "avs-cli" target, i.e. the standalone test CLI written in C. This is a Win32 feature, so on Linux the CLI is built unconditionally. Bump the required CMake version to 3.18 to use `check_linker_flag()`. Side note: There are versions of binutils (e.g. 2.35 as shipped in Ubuntu 21.04's mingw-w64 package) that have a differently named flag "--no-nxcompat", probably doing the same as "--disable-nxcompat". This flag does not seem to be in any upstream binutils release and is not tested for or supported at all. --- CMakeLists.txt | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fec9a15..af17bac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,9 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.18) project(vis_avs VERSION 2.82) include(CheckIncludeFiles) +include(CheckLinkerFlag) include(ExternalProject) file(GLOB SRC_FILES_NS_EEL @@ -129,8 +130,6 @@ add_library(avs_common OBJECT ) add_library(libavs SHARED $) -add_executable(avs-cli avs/avs-cli.c) -target_link_libraries(avs-cli libavs) check_include_files( "libavcodec/avcodec.h;libavformat/avformat.h;libswscale/swscale.h" @@ -155,7 +154,22 @@ if(WIN32) ) target_compile_definitions(vis_avs PUBLIC CAN_TALK_TO_WINAMP) - target_link_options(avs-cli PUBLIC -Wl,--disable-nxcompat) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + check_linker_flag(C "/NXCOMPAT:NO" HAS_DISABLE_NXCOMPAT) + set(NXCOMPAT_DISABLED_FLAG "/NXCOMPAT:NO") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + check_linker_flag(C "-Wl,--disable-nxcompat" HAS_DISABLE_NXCOMPAT) + set(NXCOMPAT_DISABLED_FLAG "-Wl,--disable-nxcompat") + endif() + if(HAS_DISABLE_NXCOMPAT) + add_executable(avs-cli avs/avs-cli.c) + target_link_libraries(avs-cli libavs) + target_link_options(avs-cli PUBLIC ${NXCOMPAT_DISABLED_FLAG}) + else() + message(WARNING "Linker flag --disable-nxcompat not supported." + " No standalone Win32 C/C++ binaries possible (libs will work)." + " Needs binutils 2.36 or higher.") + endif() target_link_libraries(vis_avs ddraw # draw.cpp includes ddraw.h # https://docs.microsoft.com/en-us/windows/win32/api/ddraw/nf-ddraw-directdrawcreate @@ -195,6 +209,8 @@ elseif(LINUX) target_link_libraries(libavs ${PIPEWIRE_LIBRARIES}) target_link_options(libavs PUBLIC -Wl,-m elf_i386) set(AVS_DYLIB_EXTENSION "so") + add_executable(avs-cli avs/avs-cli.c) + target_link_libraries(avs-cli libavs) endif()