Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support for GPU acceleration (CUDA) on recovery file creation. #176

Open
wants to merge 27 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
004115b
pw: create initial cuda galois files
RisaKirisu Jul 1, 2022
42d8881
pw: add profiling workspace to gitignore
RisaKirisu Jul 1, 2022
370f9c4
pw: Adding galois arithmatics CUDA routines.
RisaKirisu Jul 8, 2022
ce94b0c
pw: reconsider code structure.
RisaKirisu Jul 21, 2022
d5beefc
pw: CUDA Galois class WIP.
RisaKirisu Jul 21, 2022
f089df6
pw: Galois_cu.cuh finished.
RisaKirisu Jul 23, 2022
9d79147
pw: Update .gitignore
RisaKirisu Jul 23, 2022
3612177
pw: galosi_cu unit test WIP.
RisaKirisu Jul 24, 2022
795cb46
pw: CUDA Galois test mult finished.
RisaKirisu Jul 27, 2022
2a33541
pw: Galois field arithmetics tested GOOD
RisaKirisu Jul 31, 2022
e4287a6
pw: WIP Cuda kernel for a cuda version of ReedSolomon::Process.
RisaKirisu Aug 5, 2022
a744848
pw: reedsolomon process kernel finished.
RisaKirisu Aug 12, 2022
133a216
pw: CUDA Reedsolomon Process function WIP
RisaKirisu Aug 15, 2022
0c413fc
pw: ReedSolomon sequential kernel tested
RisaKirisu Aug 17, 2022
ebda062
pw: reedsolomon: reduce kernel completed and tested
RisaKirisu Aug 20, 2022
a4509f3
pw: modified reedsolomen cuda route to use cudaStream for concurrent …
RisaKirisu Sep 15, 2022
3a1f874
pw: integrate cuda accelerated creating route into the program
RisaKirisu Sep 24, 2022
6df08e8
pw: Convert to CMake build system
RisaKirisu Oct 2, 2022
e889309
pw: added cmake code for compiling cuda files. fixed a bug where inpu…
RisaKirisu Oct 10, 2022
89375e8
pw: fixed a bug where last chunks of recovery blocks are written inco…
RisaKirisu Oct 15, 2022
8b5ca60
Update README.md for cuda support
RisaKirisu Nov 10, 2022
74207e6
pw: add install to CMakeLists. Enable longmultiply (was commented out…
RisaKirisu Nov 10, 2022
f36dec7
pw: add copyright stuff
RisaKirisu Nov 10, 2022
60e13ab
pw: copyright stuff
RisaKirisu Nov 10, 2022
798fad5
pw: copyright stuff
RisaKirisu Nov 10, 2022
c568fea
pw: copyright stuff
RisaKirisu Nov 10, 2022
41494cb
pw: update program version and correct CUDA architecture version for …
RisaKirisu Nov 10, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,14 @@ perf.*
.#*
\#*#
.~lock.*#

# vscode settings
.vscode/settings
.vscode/
src/.vscode

# profiling workspace
profiling/

# cmake build dir
build_cmake/
87 changes: 87 additions & 0 deletions CMakeConfig.h.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
*/
#cmakedefine HAVE_DIRENT_H 1

/* Define to 1 if you have the <endian.h> header file. */
#cmakedefine HAVE_ENDIAN_H 1

/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */
#cmakedefine HAVE_FSEEKO 1

/* Define to 1 if you have the `getopt' function. */
#cmakedefine HAVE_GETOPT 1

/* Define to 1 if you have the <getopt.h> header file. */
#cmakedefine HAVE_GETOPT_H 1

/* Define to 1 if you have the `getopt_long' function. */
#cmakedefine HAVE_GETOPT_LONG 1

/* Define to 1 if you have the <inttypes.h> header file. */
#cmakedefine HAVE_INTTYPES_H 1

/* Define to 1 if you have the <limits.h> header file. */
#cmakedefine HAVE_LIMITS_H 1

/* Define to 1 if you have the `memcpy' function. */
#cmakedefine HAVE_MEMCPY 1

/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
#cmakedefine HAVE_NDIR_H 1

/* Define to 1 if stdbool.h conforms to C99. */
#cmakedefine HAVE_STDBOOL_H 1

/* Define to 1 if you have the <stdint.h> header file. */
#cmakedefine HAVE_STDINT_H 1

/* Define to 1 if you have the <stdio.h> header file. */
#cmakedefine HAVE_STDIO_H 1

/* Define to 1 if you have the <stdlib.h> header file. */
#cmakedefine HAVE_STDLIB_H 1

/* Define to 1 if you have the `strcasecmp' function. */
#cmakedefine HAVE_STRCASECMP 1

/* Define to 1 if you have the `strchr' function. */
#cmakedefine HAVE_STRCHR 1

/* Define to 1 if you have the `stricmp' function. */
#cmakedefine HAVE_STRICMP 1

/* Define to 1 if you have the <strings.h> header file. */
#cmakedefine HAVE_STRINGS_H 1

/* Define to 1 if you have the <string.h> header file. */
#cmakedefine HAVE_STRING_H 1

/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
*/
#cmakedefine HAVE_SYS_DIR_H 1

/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
*/
#cmakedefine HAVE_SYS_NDIR_H 1

/* Define to 1 if you have the <sys/stat.h> header file. */
#cmakedefine HAVE_SYS_STAT_H 1

/* Define to 1 if you have the <sys/types.h> header file. */
#cmakedefine HAVE_SYS_TYPES_H 1

/* Define to 1 if you have the <unistd.h> header file. */
#cmakedefine HAVE_UNISTD_H 1

/* Define to 1 if all of the C90 standard headers exist (not just the ones
required in a freestanding environment). This macro is provided for
backward compatibility; new code need not use it. */
#cmakedefine STDC_HEADERS 1

#cmakedefine ENABLE_CUDA 1

/* Name of package */
#define PACKAGE "@PROJECT_NAME@"

/* Version number of package */
#define VERSION "@PROJECT_VERSION@"
260 changes: 260 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
cmake_minimum_required(VERSION 3.23)

project(par2cmdline VERSION 0.9.0 LANGUAGES CXX)

# Optionally enable CUDA
option(ENABLE_CUDA "Enable CUDA" OFF)

if (ENABLE_CUDA)
# Check if CUDA is available
include(CheckLanguage)
check_language(CUDA)
if (CMAKE_CUDA_COMPILER)
enable_language(CUDA)
find_package(CUDA REQUIRED)
include_directories("${CUDA_INCLUDE_DIRS}")
else ()
message(STATUS "No CUDA compiler found")
endif ()
endif (ENABLE_CUDA)

set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CUDA_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED True)
set(CMAKE_CUDA_STANDARD_REQUIRED True)

if(NOT (CMAKE_BUILD_TYPE OR DEFINED ENV{CMAKE_BUILD_TYPE}))
set(CMAKE_BUILD_TYPE Release)
endif(NOT (CMAKE_BUILD_TYPE OR DEFINED ENV{CMAKE_BUILD_TYPE}))
message(STATUS ${CMAKE_BUILD_TYPE})

# Check existing headers and generate config.h
# for multi-platform support
include(CheckIncludeFiles)
include(CheckSymbolExists)
include(CheckFunctionExists)

check_include_files(dirent.h HAVE_DIRENT_H)
check_include_files(endian.h HAVE_ENDIAN_H)
check_include_files(getopt.h HAVE_GETOPT_H)
check_include_files(inttypes.h HAVE_INTTYPES_H)
check_include_files(limits.h HAVE_LIMITS_H)
check_include_files(ndir.h HAVE_NDIR_H)
check_include_files(stdbool.h HAVE_STDBOOL_H)
check_include_files(stdint.h HAVE_STDINT_H)
check_include_files(stdio.h HAVE_STDIO_H)
check_include_files(stdlib.h HAVE_STDLIB_H)
check_include_files(strings.h HAVE_STRINGS_H)
check_include_files(string.h HAVE_STRING_H)
check_include_files(sys/dir.h HAVE_SYS_DIR_H)
check_include_files(sys/ndir.h HAVE_SYS_NDIR_H)
check_include_files(sys/stat.h HAVE_SYS_STAT_H)
check_include_files(sys/types.h HAVE_SYS_TYPES_H)
check_include_files(unistd.h HAVE_UNISTD_H)

check_include_files("stdlib.h;stdarg.h;string.h;float.h" StandardHeadersExist)
if(StandardHeadersExist)

check_function_exists(memchr memchrExists)
if(memchrExists)

check_symbol_exists(free stdlib.h freeExists)
if(freeExists)

message(STATUS "ANSI C header files - found")
set(STDC_HEADERS 1 CACHE INTERNAL "System has ANSI C header files")

endif(freeExists)
endif(memchrExists)
endif(StandardHeadersExist)

check_function_exists(fseeko HAVE_FSEEKO)
check_function_exists(getopt HAVE_GETOPT)
check_function_exists(getopt_long HAVE_GETOPT_LONG)
check_function_exists(memcpy HAVE_MEMCPY)
check_function_exists(strcasecmp HAVE_STRCASECMP)
check_function_exists(strchr HAVE_STRCHR)
check_function_exists(stricmp HAVE_STRICMP)

# replacement for AC_C_BIGENDIAN
include (TestBigEndian)
test_big_endian(WORDS_BIGENDIAN)

# generate config.h
configure_file(CMakeConfig.h.in ${PROJECT_BINARY_DIR}/config.h)

add_compile_options(
-Wall -pipe -fstack-protector-strong
# $<$<CONFIG:RELEASE>:-O3>
# $<$<CONFIG:DEBUG>:-O0>
# $<$<CONFIG:DEBUG>:-g>
)

# compile targets
set(CMAKE_STATIC_LIBRARY_PREFIX "")
set(LIBPAR2SRC
src/crc.cpp
src/creatorpacket.cpp
src/criticalpacket.cpp
src/datablock.cpp
src/descriptionpacket.cpp
src/diskfile.cpp
src/filechecksummer.cpp
src/galois.cpp
src/mainpacket.cpp
src/md5.cpp
src/par1fileformat.cpp
src/par1repairer.cpp
src/par1repairersourcefile.cpp
src/par2creator.cpp
src/par2creatorsourcefile.cpp
src/par2fileformat.cpp
src/par2repairer.cpp
src/par2repairersourcefile.cpp
src/recoverypacket.cpp
src/reedsolomon.cpp
src/verificationhashtable.cpp
src/verificationpacket.cpp
src/libpar2.cpp
)

set(PAR2SRC
src/par2cmdline.cpp
src/commandline.cpp
)

# add cuda source files if enabled cuda
if (ENABLE_CUDA AND NOT "${CMAKE_CUDA_COMPILER}" STREQUAL "")
set(LIBPAR2SRC
${LIBPAR2SRC}
src/galois_cu.cu
src/par2creator.cu
src/reedsolomon.cu
)
# set_source_files_properties(${PAR2SRC} PROPERTIES LANGUAGE CUDA)
# message(STATUS "${LIBPAR2SRC}")

endif (ENABLE_CUDA AND NOT "${CMAKE_CUDA_COMPILER}" STREQUAL "")

# build libpar2
add_library(libpar2 STATIC ${LIBPAR2SRC})

# build par2
add_executable(par2 ${PAR2SRC})

add_compile_definitions(HAVE_CONFIG_H)

include_directories(${CMAKE_BINARY_DIR})

target_link_libraries(par2 PRIVATE libpar2)

# If OpenMP is installed, link against openmp
find_package(OpenMP)
if (OPENMP_CXX_FOUND)
target_compile_options(libpar2 PRIVATE ${OpenMP_CXX_FLAGS})
target_link_libraries(par2 PRIVATE OpenMP::OpenMP_CXX)
target_link_libraries(libpar2 PRIVATE OpenMP::OpenMP_CXX)
endif()

if (ENABLE_CUDA AND NOT "${CMAKE_CUDA_COMPILER}" STREQUAL "")
set_target_properties(libpar2 PROPERTIES CUDA_ARCHITECTURES "all-major")
endif (ENABLE_CUDA AND NOT "${CMAKE_CUDA_COMPILER}" STREQUAL "")

# ===========================
# Testing
enable_testing()

# Build unit tests
add_executable(letype_test src/letype_test.cpp)
add_executable(crc_test src/crc_test.cpp src/crc.cpp)
add_executable(md5_test src/md5_test.cpp src/md5.cpp)
add_executable(diskfile_test src/diskfile_test.cpp src/diskfile.cpp)
add_executable(libpar2_test src/libpar2_test.cpp)
target_link_libraries(libpar2_test PRIVATE libpar2)

add_executable(commandline_test src/commandline_test.cpp src/commandline.cpp)
target_link_libraries(commandline_test PRIVATE libpar2)

add_executable(descriptionpacket_test src/descriptionpacket_test.cpp src/descriptionpacket.cpp)
target_link_libraries(descriptionpacket_test PRIVATE libpar2)

add_executable(criticalpacket_test src/criticalpacket_test.cpp src/criticalpacket.cpp)
target_link_libraries(criticalpacket_test PRIVATE libpar2)

add_executable(reedsolomon_test src/reedsolomon_test.cpp src/reedsolomon.cpp)
add_executable(galois_test src/galois_test.cpp src/galois.cpp)

set(TEST_OUTPUT_DIR ${CMAKE_BINARY_DIR}/tests)
set_target_properties(
letype_test
crc_test
md5_test
diskfile_test
libpar2_test
commandline_test
descriptionpacket_test
criticalpacket_test
reedsolomon_test
galois_test
PROPERTIES
RUNTIME_OUTPUT_DIRECTORY ${TEST_OUTPUT_DIR}
)

# Run tests
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/tests DESTINATION ${CMAKE_BINARY_DIR})
find_program (BASH_PROGRAM bash)
if (BASH_PROGRAM)

add_test(NAME test1 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test1)
add_test(NAME test2 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test2)
add_test(NAME test3 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test3)
add_test(NAME test4 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test4)
add_test(NAME test5 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test5)
add_test(NAME test6 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test6)
add_test(NAME test7 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test7)
add_test(NAME test8 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test8)
add_test(NAME test9 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test9)
add_test(NAME test10 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test10)
add_test(NAME test11 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test11)
add_test(NAME test12 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test12)
add_test(NAME test13 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test13)
add_test(NAME test14 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test14)
add_test(NAME test15 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test15)
add_test(NAME test16 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test16)
add_test(NAME test17 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test17)
add_test(NAME test18 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test18)
add_test(NAME test19 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test19)
add_test(NAME test20 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test20)
add_test(NAME test21 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test21)
add_test(NAME test22 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test22)
add_test(NAME test23 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test23)
add_test(NAME test24 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test24)
add_test(NAME test25 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test25)
add_test(NAME test26 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test26)
add_test(NAME test27 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test27)
add_test(NAME test28 COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/test28)
add_test(NAME unit_tests COMMAND ${BASH_PROGRAM} ${CMAKE_BINARY_DIR}/tests/unit_tests)

endif (BASH_PROGRAM)

# ==========================
# install
message(STATUS "Install prefix: ${CMAKE_INSTALL_PREFIX}")
install(TARGETS par2
DESTINATION bin)

add_custom_command(
TARGET par2
POST_BUILD
COMMAND ln;-sf;par2${CMAKE_EXECUTABLE_SUFFIX};par2create${CMAKE_EXECUTABLE_SUFFIX}
COMMAND ln;-sf;par2${CMAKE_EXECUTABLE_SUFFIX};par2verify${CMAKE_EXECUTABLE_SUFFIX}
COMMAND ln;-sf;par2${CMAKE_EXECUTABLE_SUFFIX};par2repair${CMAKE_EXECUTABLE_SUFFIX}
)

install(
FILES
${CMAKE_CURRENT_BINARY_DIR}/par2create${CMAKE_EXECUTABLE_SUFFIX}
${CMAKE_CURRENT_BINARY_DIR}/par2verify${CMAKE_EXECUTABLE_SUFFIX}
${CMAKE_CURRENT_BINARY_DIR}/par2repair${CMAKE_EXECUTABLE_SUFFIX}
DESTINATION bin
)
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,16 @@ For macOS you can install llvm via homebrew to get OpenMP support.

See *INSTALL* for full details on how to use the *configure* script.

## Compiling par2cmdline with CUDA (NVIDIA GPU) acceleration
To compile with CUDA option enabled on Linux and other Unix variantes, use the following commands:

mkdir build_cmake
cd build_cmake
cmake -DENABLE_CUDA=ON ..
make
make check
make install

## Using par2cmdline

The command line parameters for par2cmdline are as follow:
Expand Down Expand Up @@ -101,6 +111,7 @@ The command line parameters for par2cmdline are as follow:
-n<n> : Number of recovery files (don't use both -n and -l)
-m<n> : Memory (in MB) to use
-t<n> : Number of threads to use (Auto-detected)
-C : Use CUDA device to accelerate recovery files creation
-v [-v] : Be more verbose
-q [-q] : Be more quiet (-qq gives silence)
-p : Purge backup files and par files on successful recovery or
Expand Down
Loading