From 369a351fa17cc9d9ef2112402d9abdda97321e97 Mon Sep 17 00:00:00 2001 From: Carsten Teibes Date: Tue, 16 Apr 2024 00:45:48 +0200 Subject: [PATCH] WIP: zip support --- CMakeLists.txt | 2 +- ext/CMakeLists.txt | 18 +- ext/mg/assetsys.cpp | 2 + ext/mg/assetsys.h | 1680 ++++++++++++++++++++++++++ ext/mg/strpool.cpp | 2 + ext/mg/strpool.h | 1313 ++++++++++++++++++++ ext/miniz/miniz.h | 9 +- src/game/game.h | 2 +- src/io/file.cpp | 275 +++-- src/io/file.h | 20 +- src/io/gfx/font.cpp | 4 +- src/io/sound.cpp | 4 +- src/jj1/bonuslevel/jj1bonuslevel.cpp | 2 +- src/jj1/level/jj1levelload.cpp | 4 +- src/jj1/scene/jj1scene.cpp | 2 +- src/jj1/scene/jj1scene.h | 5 +- src/jj1/scene/jj1sceneload.cpp | 16 +- src/menu/mainmenu.cpp | 2 +- 18 files changed, 3253 insertions(+), 109 deletions(-) create mode 100644 ext/mg/assetsys.cpp create mode 100644 ext/mg/assetsys.h create mode 100644 ext/mg/strpool.cpp create mode 100644 ext/mg/strpool.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 25557fad..e6dc7db2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,7 +265,7 @@ set_property(SOURCE src/main.cpp PROPERTY COMPILE_DEFINITIONS OJ_URL="${PROJECT_HOMEPAGE_URL}"; OJ_BUGREPORT="${OJ_BUGREPORT}") target_link_libraries(OpenJazz - argparse psmplug stb lodepng miniz + argparse psmplug stb lodepng mg miniz ${OJ_LIBS_SCALE} ${OJ_LIBS_SDL} ${OJ_LIBS_NET} ${OJ_LIBS_HOST}) # platform stuff diff --git a/ext/CMakeLists.txt b/ext/CMakeLists.txt index 2dfd3a25..ae58095e 100644 --- a/ext/CMakeLists.txt +++ b/ext/CMakeLists.txt @@ -30,7 +30,7 @@ target_include_directories(stb INTERFACE stb) # miniz add_library(miniz STATIC miniz/miniz.h miniz/miniz.cpp) -target_compile_definitions(miniz INTERFACE MINIZ_NO_ARCHIVE_APIS) +target_compile_definitions(miniz INTERFACE MINIZ_NO_ZLIB_COMPATIBLE_NAMES) #target_compile_definitions(miniz PRIVATE _LARGEFILE64_SOURCE) target_include_directories(miniz INTERFACE miniz) @@ -51,6 +51,22 @@ target_compile_definitions(lodepng INTERFACE target_include_directories(lodepng INTERFACE lodepng) target_link_libraries(lodepng PRIVATE miniz) +# mg + +add_library(mg STATIC + mg/assetsys.cpp + mg/assetsys.h + mg/strpool.cpp + mg/strpool.h +) +#target_compile_definitions(mg INTERFACE +# ASSETSYS_U64 uint64_t +# STRPOOL_U32 uint32_t +# STRPOOL_U64 uint64_t +#) +target_include_directories(mg INTERFACE mg) +target_link_libraries(mg PRIVATE miniz) + if(ANDROID) include(FetchContent) diff --git a/ext/mg/assetsys.cpp b/ext/mg/assetsys.cpp new file mode 100644 index 00000000..d95d01c7 --- /dev/null +++ b/ext/mg/assetsys.cpp @@ -0,0 +1,2 @@ +#define ASSETSYS_IMPLEMENTATION +#include "assetsys.h" diff --git a/ext/mg/assetsys.h b/ext/mg/assetsys.h new file mode 100644 index 00000000..19f9fda1 --- /dev/null +++ b/ext/mg/assetsys.h @@ -0,0 +1,1680 @@ +/* +------------------------------------------------------------------------------ + Licensing information can be found at the end of the file. +------------------------------------------------------------------------------ + +assetsys.h - v1.5 - File system abstraction to read from zip-files, for C/C++. + +Do this: + #define ASSETSYS_IMPLEMENTATION +before you include this file in *one* C/C++ file to create the implementation. + +If you are using miniz.c for other things, and have included it in your +project, you need to define ASSETSYS_NO_MINIZ as well, to avoid duplicate +definitions. + +Dependencies: + strpool.h +*/ + +#ifndef assetsys_h +#define assetsys_h + +// OpenJazz Modifications +#include +#include +#define ASSETSYS_U64 uint64_t +// End of Openjazz Modifications + +#ifndef ASSETSYS_U64 + #define ASSETSYS_U64 unsigned long long +#endif + +typedef enum assetsys_error_t + { + ASSETSYS_SUCCESS = 0, + ASSETSYS_ERROR_INVALID_PATH = -1, + ASSETSYS_ERROR_INVALID_MOUNT = -2, + ASSETSYS_ERROR_FAILED_TO_READ_ZIP = -3, + ASSETSYS_ERROR_FAILED_TO_CLOSE_ZIP = -4, + ASSETSYS_ERROR_FAILED_TO_READ_FILE = -5, + ASSETSYS_ERROR_FILE_NOT_FOUND = -6, + ASSETSYS_ERROR_DIR_NOT_FOUND = -7, + ASSETSYS_ERROR_INVALID_PARAMETER = -8, + ASSETSYS_ERROR_BUFFER_TOO_SMALL = -9, + } assetsys_error_t; + +typedef struct assetsys_t assetsys_t; + +assetsys_t* assetsys_create( void* memctx ); +void assetsys_destroy( assetsys_t* sys ); + +assetsys_error_t assetsys_mount( assetsys_t* sys, char const* path, char const* mount_as ); +assetsys_error_t assetsys_mount_from_memory( assetsys_t* sys, void const* data, int size, char const* mount_as); +assetsys_error_t assetsys_dismount( assetsys_t* sys, char const* path, char const* mounted_as ); + +typedef struct assetsys_file_t { ASSETSYS_U64 mount; ASSETSYS_U64 path; int index; } assetsys_file_t; + +assetsys_error_t assetsys_file( assetsys_t* sys, char const* path, assetsys_file_t* file ); +assetsys_error_t assetsys_file_load( assetsys_t* sys, assetsys_file_t file, int* size, void* buffer, int capacity ); +int assetsys_file_size( assetsys_t* sys, assetsys_file_t file ); + +int assetsys_file_count( assetsys_t* sys, char const* path ); +char const* assetsys_file_name( assetsys_t* sys, char const* path, int index ); +char const* assetsys_file_path( assetsys_t* sys, char const* path, int index ); + +int assetsys_subdir_count( assetsys_t* sys, char const* path ); +char const* assetsys_subdir_name( assetsys_t* sys, char const* path, int index ); +char const* assetsys_subdir_path( assetsys_t* sys, char const* path, int index ); + +// OpenJazz addition +const char *assetsys_file_to_path(assetsys_t* sys, assetsys_file_t file); + +#endif /* assetsys_h */ + +/** + +assetsys.h +========== + +File system abstraction to read from zip-files, for C/C++. + + +Example +------- + + #define ASSETSYS_IMPLEMENTATION + #include "assetsys.h" + + #define STRPOOL_IMPLEMENTATION + #include "strpool.h" + + #include // for printf + + void list_assets( assetsys_t* assetsys, char const* path, int indent ) { + // Print folder names and recursively list assets + for( int i = 0; i < assetsys_subdir_count( assetsys, path ); ++i ) { + char const* subdir_name = assetsys_subdir_name( assetsys, path, i ); + for( int j = 0; j < indent; ++j ) printf( " " ); + printf( "%s/\n", subdir_name ); + + char const* subdir_path = assetsys_subdir_path( assetsys, path, i ); + list_assets( assetsys, subdir_path, indent + 1 ); + } + + // Print file names + for( int i = 0; i < assetsys_file_count( assetsys, path ); ++i ) { + char const* file_name = assetsys_file_name( assetsys, path, i ); + for( int j = 0; j < indent; ++j ) printf( " " ); + printf( "%s\n", file_name ); + } + } + + int main( void ) { + assetsys_t* assetsys = assetsys_create( 0 ); + + // Mount current working folder as a virtual "/data" path + assetsys_mount( assetsys, ".", "/data" ); + + // Print all files and subfolders + list_assets( assetsys, "/", 0 ); // Start at root + + // Load a file + assetsys_file_t file; + assetsys_file( assetsys, "/data/readme.txt", &file ); + int size = assetsys_file_size( assetsys, file ); + char* content = (char*) malloc( size + 1 ); // extra space for '\0' + int loaded_size = 0; + assetsys_file_load( assetsys, file, &loaded_size, content, size ); + content[ size ] = '\0'; // zero terminate the text file + printf( "%s\n", content ); + free( content ); + + assetsys_destroy( assetsys ); + } + + +API Documentation +----------------- + +assetsys.h is a system for loading binary assets into your game. It allows you to mount directories and archive files +(bundles of files; assetsys.h supports using standard zip files for this) assign them a virtual path. You then load +assets through assetsys using the virtual path (which can stay the same even if the mounts change). assetsys.h is +case-insensitive regardless of platform, and only accepts forward slash "/" as path separator, to ensure consistent +behavior. It allows you to mount several paths or archive files to the same virtual path, even if they contain files of +the same name, and a later mount will take precedence over a previous one. This allows you to, for example, have a +`data` archive file and an `update` archive file, where `update` contains new versions of only some of the files. + +assetsys.h is a single-header library, and does not need any .lib files or other binaries, or any build scripts. To use +it, you just include assetsys.h to get the API declarations. To get the definitions, you must include assetsys.h from +*one* single C or C++ file, and #define the symbol `ASSETSYS_IMPLEMENTATION` before you do. + +assetsys.h has a dependency om another single-header library, strpool.h, which is used for efficient storage and +comparison of the filename and path strings. assetsys.h automatically includes strpool.h, so it must reside in the same +path. It does not specify the STRPOOL_IMPLEMENTATION define, on the assumption that you might be including strpool.h in +some other part of your program. If you are not, you can make assetsys.h include the strpool implemention by doing: + + #define ASSETSYS_IMPLEMENTATION + #define STRPOOL_IMPLEMENTATION + #include "assetsys.h" + + +### Customization + +There are a few different things in assetsys.h which are configurable by #defines. Most of the API use the `int` data +type, for integer values where the exact size is not important. However, for some functions, it specifically makes use +of an unsigned 64 bit data types. It default to using `unsigned long long`, but can be redefined by #defining +ASSETSYS_U64, before including assetsys.h. This is useful if you, for example, use the types from `` in the +rest of your program, and you want assetsys.h to use compatible types. In this case, you would include assetsys.h using +the following code: + + #define ASSETSYS_U64 uint64_t + #include "assetsys.h" + +Note that when customizing the data type, you need to use the same definition in every place where you include +assetsys.h, as they affect the declarations as well as the definitions. + +The rest of the customizations only affect the implementation, so will only need to be defined in the file where you +have the #define ASSETSYS_IMPLEMENTATION. + + +#### Custom memory allocators + +To store the internal data structures, ini.h needs to do dynamic allocation by calling `malloc`. Programs might want to +keep track of allocations done, or use custom defined pools to allocate memory from. assetsys.h allows for specifying +custom memory allocation functions for `malloc` and `free`. This is done with the following code: + + #define ASSETSYS_IMPLEMENTATION + #define ASSETSYS_MALLOC( ctx, size ) ( my_custom_malloc( ctx, size ) ) + #define ASSETSYS_FREE( ctx, ptr ) ( my_custom_free( ctx, ptr ) ) + #include "assetsys.h" + +where `my_custom_malloc` and `my_custom_free` are your own memory allocation/deallocation functions. The `ctx` parameter +is an optional parameter of type `void*`. When `assetsys_init` is called, you can set the `memctx` field of the `config` +parameter, to a pointer to anything you like, and which will be passed through as the `ctx` parameter to every +`ASSETSYS_MALLOC`/`ASSETSYS_FREE` call. For example, if you are doing memory tracking, you can pass a pointer to your +tracking data as `memctx`, and in your custom allocation/deallocation function, you can cast the `ctx` param back to the +right type, and access the tracking data. + +If no custom allocator is defined, assetsys.h will default to `malloc` and `free` from the C runtime library. + + +#### Custom assert + +assetsys.h makes use of asserts to report usage errors and code errors. By default, it makes use of the C runtime +library `assert` macro, which only executes in debug builds. However, it allows for substituting with your own assert +function or macro using the following code: + + #define ASSETSYS_IMPLEMENTATION + #define ASSETSYS_ASSERT( condition ) ( my_custom_assert( condition ) ) + #include "assetsys.h" + +Note that if you only want the asserts to trigger in debug builds, you must add a check for this in your custom assert. + + +#### miniz implementation + +assetsys.h makes use of the miniz library for parsing and decompressing zip files. It includes the entire miniz source +code inside assetsys.h, so normally you don't have to worry about it. However, in the case where you might already be +using miniz in some other part of the program, you can tell assetsys.h to not include the implementation for miniz. It +will still include the miniz definitions, and if you don't include the miniz implementation elsewhere, you will get a +linker error. To exclude the miniz implementation, simply define `ASSETSYS_NO_MINIZ` before including assetsys.h, like +this: + + #define ASSETSYS_IMPLEMENTATION + #define ASSETSYS_NO_MINIZ + #include "assetsys.h" + + +assetsys_create +--------------- + + assetsys_t* assetsys_create( void* memctx ) + +Creates a new assetsys instance. assetsys.h does not use any global variables, all data it needs is accessed through +the instance created by calling assetsys_create. Different instances can be used safely from different threads, but if +using the same instance from multiple threads, it is up to the user to make sure functions are not called concurrently, +for example by adding a mutex lock around each call. + + +assetsys_destroy +---------------- + + void assetsys_destroy( assetsys_t* sys ) + +Destroys an assetsys instance, releasing all the resources used by it. + + +assetsys_mount +-------------- + + assetsys_error_t assetsys_mount( assetsys_t* sys, char const* path, char const* mount_as ) + +Mounts the data source `path`, making all its files accessible through this assetsys instance. The data source can be +either a folder or an archive file (a standard .zip file, with or without compression). `path` must use forward slash +`/` as path separator, never backslash, regardless of platform. It must not end with a path separator. The string +`mount_as` will be prepended to all mounted files, and can be passed as "/" to mount as root. `mount_as` may +not contain the characters backslash `\` or colon `:`. It must not end with a path separator (unless consisting of +a single path separator only, "/"). `assetsys_mount` will return `ASSETSYS_ERROR_INVALID_PARAMETER` if either `path` +or `mount_as` is NULL. It will return `ASSETSYS_ERROR_INVALID_PATH` if the conditions detailed above are not met, or +if the file or folder specified by `path` could not be found. If `path` indicates a file, and it is not a valid archive +file, `assetsys_mount` returns `ASSETSYS_ERROR_FAILED_TO_READ_ZIP`. + +If multiple mounts contains the same file and it is accessible through the same full path (whether because of the +`mount_as` prefix or not), the last mounted data source will be used when loading that file. + + +assetsys_mount_from_memory +-------------------------- + + assetsys_error_t assetsys_mount_from_memory( assetsys_t* sys, void const* data, int size, char const* mount_as ) + +Same as `assetsys_mount()`, but takes a data buffer of an archived *.zip* file, along with the size of the file. + + +assetsys_dismount +----------------- + + assetsys_error_t assetsys_dismount( assetsys_t* sys, char const* path, char const* mounted_as ) + +Removes a data source which was mounted by calling `assetsys_mount`. `path` and `mounted_as` must be the same as was +used when mounting. If `path` is NULL, `assetsys_dismount` returns `ASSETSYS_ERROR_INVALID_PARAMETER`. If `mounted_as` +is NULL, or no matching mount could be found, it returns `ASSETSYS_ERROR_INVALID_MOUNT`. + + +assetsys_file +------------- + + assetsys_error_t assetsys_file( assetsys_t* sys, char const* path, assetsys_file_t* file ) + +Retrieves a handle for the file specified by `path`. `path` needs to be an absolute path, including the `mount_as` +prefix specified when the data source was mounted, and matching is case insensitive. The mounts are searched in reverse +order they were added, and if a file with the specified path could not be found, `assetsys_file` returns +`ASSETSYS_ERROR_FILE_NOT_FOUND`. The handle is written to `file`, which must be a pointer to a `assetsys_file_t` +variable declared by the caller. The handle is used in calls to `assetsys_file_load` and `assetsys_file_size`. The +handle is only valid until any mounts are modified by calling `assetsys_mount` or `assetsys_dismount`. + + +assetsys_file_load +------------------ + + assetsys_error_t assetsys_file_load( assetsys_t* sys, assetsys_file_t file, int* size, void* buffer, int capacity ) + +Load the data from the file specified by the handle `file` (initialized by calling `assetsys_file`) and writes it into +the memory indicated by `buffer`. This memory buffer must be large enough to fit the entire file, and the `capacity` +parameter must indicate its size. To find out how large the buffer needs to be, call `assetsys_file_size`. The size of +the file will also be reported in the `size` parameter, unless it is passed in as NULL. Note that the two sizes can be +reported with different values if the file was updated on disk between the call to `assetsys_file_size` and the call to +`assetsys_file_load`. +If the file could not be loaded, `assetsys_file_load` returns `ASSETSYS_ERROR_FAILED_TO_READ_FILE`. If the `capacity` +parameter is too small to hold the file data, `assetsys_file_load` returns `ASSETSYS_ERROR_BUFFER_TOO_SMALL`. + + +assetsys_file_size +------------------ + + int assetsys_file_size( assetsys_t* sys, assetsys_file_t file ) + +Returns the size, in bytes, of the file specified by the handle `file` (initialized by calling `assetsys_file`). If the +file handle is not valid, `assetsys_file_size` returns 0. If the file is found in a directory mount, the size will be +re-queried on each call to `assetsys_file_size` (to support the case where a file have been re-saved to disk since the +last call). In the case where the file resides in an archive mount, `assetsys_file_size` will return its initial value. + + +assetsys_file_count +------------------- + + int assetsys_file_count( assetsys_t* sys, char const* path ) + +Returns the number of files in the directory with the specified path, or 0 if the path is invalid. `path` needs to be an +absolute path, including the `mount_as` prefix specified when the data source was mounted, and matching is case +insensitive. `assetsys_file_count` returns the total number of files from all mounts, which fall under the path. + + +assetsys_file_name +------------------ + + char const* assetsys_file_name( assetsys_t* sys, char const* path, int index ) + +Returns the filename and extension (but not the full path) of one of the files in the specified path. `path` needs to be +an absolute path, including the `mount_as` prefix specified when the data source was mounted, and matching is case +insensitive. `index` needs to be between 0 and one less than the count returned by calling `assetsys_file_count` with +the same path. If the path is invalid or index is out of range, `assetsys_file_name` returns NULL. + + +assetsys_file_path +------------------ + + char const* assetsys_file_path( assetsys_t* sys, char const* path, int index ) + +Returns the full path, including filename and extension, of one of the files in the specified path. `path` needs to be +an absolute path, including the `mount_as` prefix specified when the data source was mounted, and matching is case +insensitive. `index` needs to be between 0 and one less than the count returned by calling `assetsys_file_count` with +the same path. If the path is invalid or index is out of range, `assetsys_file_path` returns NULL. + + +assetsys_subdir_count +--------------------- + + int assetsys_subdir_count( assetsys_t* sys, char const* path ) + +Returns the number of subdirectories in the directory with the specified path, or 0 if the path is invalid. `path` needs +to be an absolute path, including the `mount_as` prefix specified when the data source was mounted, and matching is case +insensitive. `assetsys_subdir_count` returns the total number of directories from all mounts, which fall under the path. + + +assetsys_subdir_name +-------------------- + + char const* assetsys_subdir_name( assetsys_t* sys, char const* path, int index ) + +Returns the name (but not the full path) of one of the subdirectories in the specified path. `path` needs to be an +absolute path, including the `mount_as` prefix specified when the data source was mounted, and matching is case +insensitive. `index` needs to be between 0 and one less than the count returned by calling `assetsys_subdir_count` with +the same path. If the path is invalid or index is out of range, `assetsys_subdir_name` returns NULL. + + +assetsys_subdir_path +-------------------- + + char const* assetsys_subdir_path( assetsys_t* sys, char const* path, int index ) + +Returns the name, including the full path, of one of the files in the specified path. `path` needs to be an absolute +path, including the `mount_as` prefix specified when the data source was mounted, and matching is case insensitive. +`index` needs to be between 0 and one less than the count returned by calling `assetsys_subdir_count` with the same +path. If the path is invalid or index is out of range, `assetsys_subdir_path` returns NULL. + + +*/ + + +// If we are running tests on windows +#if defined( ASSETSYS_RUN_TESTS ) && defined( _WIN32 ) && !defined( __TINYC__ ) + // To get file names/line numbers with meory leak detection, we need to include crtdbg.h before all other files + #define _CRTDBG_MAP_ALLOC + #include +#endif + + +/* +---------------------- + IMPLEMENTATION +---------------------- +*/ + +#ifdef ASSETSYS_IMPLEMENTATION +#undef ASSETSYS_IMPLEMENTATION + +#define _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_SECURE_NO_WARNINGS + +// OpenJazz Modification +//#ifdef ASSETSYS_NO_MINIZ + //#define MINIZ_HEADER_FILE_ONLY +//#endif /* ASSETSYS_NO_MINIZ */ + +#ifdef ASSETSYS_ASSERT + #define MZ_ASSERT( x ) ASSETSYS_ASSERT( x, "Miniz assert" ) +#endif /* ASSETSYS_ASSERT */ + +#ifdef _WIN32 + #pragma warning( push ) + #pragma warning( disable: 4619 ) // pragma warning : there is no warning number 'number' + #pragma warning( disable: 4244 ) // 'conversion' conversion from 'type1' to 'type2', possible loss of data + #pragma warning( disable: 4365 ) // 'action' conversion from 'type_1' to 'type_2', signed/unsigned mismatch + #pragma warning( disable: 4548 ) // expression before comma has no effect; expected expression with side-effect + #pragma warning( disable: 4668 ) // 'symbol' is not defined as a preprocessor macro, replacing with '0' for 'directives' +#endif + +// OpenJazz Modification +// removed whole miniz copy + +#ifdef _WIN32 + #pragma warning( pop ) +#endif + +#define _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_SECURE_NO_WARNINGS +#include +#include // FILE, fopen, fseet, SEEK_END, SEEK_SET, ftell, fread + +#include "strpool.h" + +#ifndef ASSETSYS_ASSERT + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define ASSETSYS_ASSERT( expression, message ) assert( ( expression ) && ( message ) ) +#endif + +#ifndef ASSETSYS_MALLOC + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define ASSETSYS_MALLOC( ctx, size ) ( malloc( size ) ) + #define ASSETSYS_FREE( ctx, ptr ) ( free( ptr ) ) +#endif + + +#if defined( _WIN32 ) + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #if !defined( _WIN32_WINNT ) || _WIN32_WINNT < 0x0501 + #undef _WIN32_WINNT + #define _WIN32_WINNT 0x0501 // requires Windows XP minimum + // 0x0400=Windows NT 4.0, 0x0500=Windows 2000, 0x0501=Windows XP, 0x0502=Windows Server 2003, 0x0600=Windows Vista, + // 0x0601=Windows 7, 0x0602=Windows 8, 0x0603=Windows 8.1, 0x0A00=Windows 10 +#endif + #define _WINSOCKAPI_ + #pragma warning( push ) + #pragma warning( disable: 4668 ) // 'symbol' is not defined as a preprocessor macro, replacing with '0' for 'directives' + #pragma warning( disable: 4768 ) // __declspec attributes before linkage specification are ignored + #pragma warning( disable: 4255 ) // 'function' : no function prototype given: converting '()' to '(void)' + #include + #pragma warning( pop ) + + struct assetsys_internal_dir_entry_t + { + char name[ MAX_PATH ]; + BOOL is_folder; + }; + + + struct assetsys_internal_dir_t + { + HANDLE handle; + WIN32_FIND_DATAA data; + struct assetsys_internal_dir_entry_t entry; + }; + + + static void assetsys_internal_dir_open( struct assetsys_internal_dir_t* dir, char const* path ) + { + size_t path_len = strlen( path ); + BOOL trailing_path_separator = path[ path_len - 1 ] == '\\' || path[ path_len - 1 ] == '/'; + const char* string_to_append = "*.*"; + if( path_len + strlen( string_to_append ) + ( trailing_path_separator ? 0 : 1 ) >= MAX_PATH ) return; + char search_pattern[ MAX_PATH ]; + strcpy( search_pattern, path ); + if( !trailing_path_separator ) strcat( search_pattern, "\\" ); + strcat( search_pattern, string_to_append ); + + WIN32_FIND_DATAA data; + HANDLE handle = FindFirstFileA( search_pattern, &data ); + if( handle == INVALID_HANDLE_VALUE ) return; + + dir->handle = handle; + dir->data = data; + } + + + static void assetsys_internal_dir_close( struct assetsys_internal_dir_t* dir ) + { + if( dir->handle != INVALID_HANDLE_VALUE ) FindClose( dir->handle ); + } + + + static struct assetsys_internal_dir_entry_t* assetsys_internal_dir_read( struct assetsys_internal_dir_t* dir ) + { + if( dir->handle == INVALID_HANDLE_VALUE ) return NULL; + + strcpy( dir->entry.name, dir->data.cFileName ); + dir->entry.is_folder = ( dir->data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) != 0; + + BOOL result = FindNextFileA( dir->handle, &dir->data ); + if( !result ) + { + FindClose( dir->handle ); + dir->handle = INVALID_HANDLE_VALUE; + } + + return &dir->entry; + } + + + static char const* assetsys_internal_dir_name( struct assetsys_internal_dir_entry_t* entry ) + { + return entry->name; + } + + + static int assetsys_internal_dir_is_file( struct assetsys_internal_dir_entry_t* entry ) + { + return entry->is_folder == FALSE; + } + + + static int assetsys_internal_dir_is_folder( struct assetsys_internal_dir_entry_t* entry ) + { + return entry->is_folder == TRUE; + } + + +#else + + #include + #include + + struct assetsys_internal_dir_t + { + DIR* dir; + }; + + + typedef struct assetsys_internal_dir_entry_t assetsys_internal_dir_entry_t; + + + static void assetsys_internal_dir_open( struct assetsys_internal_dir_t* dir, char const* path ) + { + dir->dir = opendir( path ); + } + + + static void assetsys_internal_dir_close( struct assetsys_internal_dir_t* dir ) + { + closedir( dir->dir ); + } + + + static assetsys_internal_dir_entry_t* assetsys_internal_dir_read( struct assetsys_internal_dir_t* dir ) + { + return (assetsys_internal_dir_entry_t*)readdir( dir->dir ); + } + + + static char const* assetsys_internal_dir_name( assetsys_internal_dir_entry_t* entry ) + { + return ( (struct dirent*)entry )->d_name; + } + + + static int assetsys_internal_dir_is_file( assetsys_internal_dir_entry_t* entry ) + { + return ( (struct dirent*)entry )->d_type == DT_REG; + } + + + static int assetsys_internal_dir_is_folder( assetsys_internal_dir_entry_t* entry ) + { + return ( (struct dirent*)entry )->d_type == DT_DIR; + } + +#endif + + +static void* assetsys_internal_mz_alloc( void* memctx, size_t items, size_t size ) + { + (void) memctx; (void) items; (void) size; + ASSETSYS_U64* p = (ASSETSYS_U64*) ASSETSYS_MALLOC( memctx, ( items * size ) + sizeof( ASSETSYS_U64 ) ); + *p = ( items * size ); + return p + 1; + } + + +static void assetsys_internal_mz_free( void* memctx, void* ptr ) + { + (void) memctx; (void) ptr; + if( !ptr ) return; + ASSETSYS_U64* p = ( (ASSETSYS_U64*) ptr ) - 1; + ASSETSYS_FREE( memctx, p ); + } + + +static void* assetsys_internal_mz_realloc( void* memctx, void* ptr, size_t items, size_t size ) + { + (void) memctx; (void) ptr; (void) items; (void) size; + if( !ptr ) return assetsys_internal_mz_alloc( memctx, items, size ); + + ASSETSYS_U64* p = ( (ASSETSYS_U64*) ptr ) - 1; + ASSETSYS_U64 prev_size = *p; + if( prev_size >= ( items * size ) ) return ptr; + + ASSETSYS_U64* new_ptr = (ASSETSYS_U64*) ASSETSYS_MALLOC( memctx, ( items * size ) + sizeof( ASSETSYS_U64 ) ); + *new_ptr = ( items * size ); + ++new_ptr; + memcpy( new_ptr, ptr, (size_t) prev_size ); + ASSETSYS_FREE( memctx, p ); + return new_ptr; + } + + +static char* assetsys_internal_dirname( char const* path ); + +struct assetsys_internal_file_t + { + int size; + int zip_index; + int collated_index; + }; + +struct assetsys_internal_folder_t + { + int collated_index; + }; + +enum assetsys_internal_mount_type_t + { + ASSETSYS_INTERNAL_MOUNT_TYPE_DIR, + ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP, + }; + +struct assetsys_internal_mount_t + { + ASSETSYS_U64 path; + ASSETSYS_U64 mounted_as; + int mount_len; + enum assetsys_internal_mount_type_t type; + mz_zip_archive zip; + + struct assetsys_internal_file_t* files; + int files_count; + int files_capacity; + + struct assetsys_internal_folder_t* dirs; + int dirs_count; + int dirs_capacity; + }; + +struct assetsys_internal_collated_t + { + ASSETSYS_U64 path; + int parent; + int ref_count; + int is_file; + }; + + +struct assetsys_t + { + void* memctx; + strpool_t strpool; + + struct assetsys_internal_mount_t* mounts; + int mounts_count; + int mounts_capacity; + + struct assetsys_internal_collated_t* collated; + int collated_count; + int collated_capacity; + + char temp[ 260 ]; + }; + + +static ASSETSYS_U64 assetsys_internal_add_string( assetsys_t* sys, char const* const str ) + { + ASSETSYS_U64 h = strpool_inject( &sys->strpool, str, (int) strlen( str ) ); + strpool_incref( &sys->strpool, h ); + return h; + } + + +static char const* assetsys_internal_get_string( assetsys_t* sys, ASSETSYS_U64 const handle ) + { + return strpool_cstr( &sys->strpool, handle ); + } + + +assetsys_t* assetsys_create( void* memctx ) + { + assetsys_t* sys = (assetsys_t*) ASSETSYS_MALLOC( memctx, sizeof( assetsys_t ) ); + sys->memctx = memctx; + + strpool_config_t config = strpool_default_config; + config.memctx = memctx; + // OpenJazz modification + config.ignore_case = 1; + strpool_init( &sys->strpool, &config ); + + sys->mounts_count = 0; + sys->mounts_capacity = 16; + sys->mounts = (struct assetsys_internal_mount_t*) ASSETSYS_MALLOC( memctx, + sizeof( *sys->mounts ) * sys->mounts_capacity ); + + sys->collated_count = 0; + sys->collated_capacity = 16384; + sys->collated = (struct assetsys_internal_collated_t*) ASSETSYS_MALLOC( memctx, + sizeof( *sys->collated ) * sys->collated_capacity ); + return sys; + } + + +void assetsys_destroy( assetsys_t* sys ) + { + while( sys->mounts_count > 0 ) + { + assetsys_dismount( sys, assetsys_internal_get_string( sys, sys->mounts[ 0 ].path ), + assetsys_internal_get_string( sys, sys->mounts[ 0 ].mounted_as ) ); + } + ASSETSYS_FREE( sys->memctx, sys->collated ); + ASSETSYS_FREE( sys->memctx, sys->mounts ); + strpool_term( &sys->strpool ); + ASSETSYS_FREE( sys->memctx, sys ); + } + + +static int assetsys_internal_register_collated( assetsys_t* sys, char const* path, int const is_file ) + { + if( path[ 0 ] == '/' && path[ 1 ] == '/' ) ++path; + + ASSETSYS_U64 handle = strpool_inject( &sys->strpool, path, (int) strlen( path ) ); + + int first_free = -1; + for( int i = 0; i < sys->collated_count; ++i ) + { + if( sys->collated[ i ].ref_count > 0 && sys->collated[ i ].path == handle ) + { + ASSETSYS_ASSERT( is_file == sys->collated[ i ].is_file, "Entry type mismatch" ); + ++sys->collated[ i ].ref_count; + strpool_discard( &sys->strpool, handle ); + return i; + } + if( sys->collated[ i ].ref_count == 0 ) first_free = i; + } + + if( first_free < 0) + { + if( sys->collated_count >= sys->collated_capacity ) + { + sys->collated_capacity *= 2; + struct assetsys_internal_collated_t* new_collated = (struct assetsys_internal_collated_t*) ASSETSYS_MALLOC( + sys->memctx, sizeof( *sys->collated ) * sys->collated_capacity ); + memcpy( new_collated, sys->collated, sizeof( *sys->collated ) * sys->collated_count ); + ASSETSYS_FREE( sys->memctx, sys->collated ); + sys->collated = new_collated; + } + first_free = sys->collated_count++; + } + + + struct assetsys_internal_collated_t* dir = &sys->collated[ first_free ]; + dir->path = handle; + strpool_incref( &sys->strpool, handle ); + dir->parent = -1; + dir->ref_count = 1; + dir->is_file = is_file; + return first_free; + } + + +static void assetsys_internal_collate_directories( assetsys_t* sys, struct assetsys_internal_mount_t* const mount ) + { + for( int i = 0; i < mount->dirs_count; ++i ) + { + struct assetsys_internal_collated_t* subdir = &sys->collated[ mount->dirs[ i ].collated_index ]; + if( subdir->parent < 0 ) + { + char const* a = assetsys_internal_get_string( sys, subdir->path ); (void) a; + char* sub_path = assetsys_internal_dirname( assetsys_internal_get_string( sys, subdir->path ) ) ; + ASSETSYS_U64 handle = strpool_inject( &sys->strpool, sub_path, (int) strlen( sub_path ) - 1 ); + for( int j = 0; j < sys->collated_count; ++j ) + { + struct assetsys_internal_collated_t* dir = &sys->collated[ j ]; + char const* b = assetsys_internal_get_string( sys, dir->path ); (void) b; + if( dir->path == handle ) + { + subdir->parent = j; + break; + } + } + if( subdir->parent < 0 ) strpool_discard( &sys->strpool, handle ); + } + } + + for( int i = 0; i < mount->files_count; ++i ) + { + struct assetsys_internal_collated_t* file = &sys->collated[ mount->files[ i ].collated_index ]; + if( file->parent < 0 ) + { + char* file_path = assetsys_internal_dirname( assetsys_internal_get_string( sys, file->path ) ) ; + ASSETSYS_U64 handle = strpool_inject( &sys->strpool, file_path, file_path[0] == '/' && file_path[1] == '\0' ? 1 : (int) strlen( file_path ) - 1 ); + for( int j = 0; j < sys->collated_count; ++j ) + { + struct assetsys_internal_collated_t* dir = &sys->collated[ j ]; + if( dir->path == handle ) + { + file->parent = j; + break; + } + } + if( file->parent < 0 ) strpool_discard( &sys->strpool, handle ); + } + } + } + + +static void assetsys_internal_recurse_directories( assetsys_t* sys, int const collated_index, + struct assetsys_internal_mount_t* const mount ) + { + char const* path = assetsys_internal_get_string( sys, sys->collated[ collated_index ].path ); + path += mount->mount_len; + if( *path == '/' ) ++path; + + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->path ) ); + strcat( sys->temp, ( *path == '\0' || *sys->temp == '\0' ) ? "" : "/" ); + strcat( sys->temp, path ); + + struct assetsys_internal_dir_t dir; + assetsys_internal_dir_open( &dir, *sys->temp == '\0' ? "." : sys->temp ); + + struct assetsys_internal_dir_entry_t* dirent; + for( dirent = assetsys_internal_dir_read( &dir ); dirent != NULL; dirent = assetsys_internal_dir_read( &dir ) ) + { + char const* name = assetsys_internal_dir_name( dirent ); + if( !name || *name == '\0' || strcmp( name, "." ) == 0 || strcmp( name, ".." ) == 0 ) continue; + int is_file = assetsys_internal_dir_is_file( dirent ); + int is_folder = assetsys_internal_dir_is_folder( dirent ); + if( is_file ) + { + char const* file_path = assetsys_internal_get_string( sys, sys->collated[ collated_index ].path ); + file_path += mount->mount_len; + if( *file_path == '/' ) ++file_path; + + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->path ) ); + strcat( sys->temp, ( *file_path == '\0' || *sys->temp == '\0' ) ? "" : "/" ); + strcat( sys->temp, file_path ); + strcat( sys->temp, *sys->temp == '\0' ? "" : "/" ); + strcat( sys->temp, name ); + + struct stat s; + if( stat( sys->temp, &s ) == 0 ) + { + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->mounted_as ) ); + if( *sys->temp && sys->temp[ strlen( sys->temp ) - 1 ] != '/' ) strcat( sys->temp, "/" ); + strcat( sys->temp, file_path ); + strcat( sys->temp, *file_path == '\0' ? "" : "/" ); + strcat( sys->temp, name ); + + if( mount->files_count >= mount->files_capacity ) + { + mount->files_capacity *= 2; + struct assetsys_internal_file_t* new_files = (struct assetsys_internal_file_t*) ASSETSYS_MALLOC( + sys->memctx, sizeof( *(mount->files) ) * mount->files_capacity ); + memcpy( new_files, mount->files, sizeof( *(mount->files) ) * mount->files_count ); + ASSETSYS_FREE( sys->memctx, mount->files ); + mount->files = new_files; + } + + struct assetsys_internal_file_t* file = &mount->files[ mount->files_count++ ]; + file->size = (int) s.st_size; + file->zip_index = -1; + file->collated_index = assetsys_internal_register_collated( sys, sys->temp, 1 ); + } + } + else if( is_folder ) + { + char const* folder_path = assetsys_internal_get_string( sys, sys->collated[ collated_index ].path ); + folder_path += mount->mount_len; + if( *folder_path == '/' ) ++folder_path; + + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->path ) ); + strcat( sys->temp, ( *folder_path == '\0' || *sys->temp == '\0' ) ? "" : "/" ); + strcat( sys->temp, folder_path ); + strcat( sys->temp, *sys->temp == '\0' ? "" : "/" ); + strcat( sys->temp, name ); + + struct stat s; + if( stat( sys->temp, &s ) == 0 ) + { + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->mounted_as ) ); + if( *sys->temp && sys->temp[ strlen( sys->temp ) - 1 ] != '/' ) strcat( sys->temp, "/" ); + strcat( sys->temp, folder_path ); + strcat( sys->temp, *folder_path == '\0' ? "" : "/" ); + strcat( sys->temp, name ); + + if( mount->dirs_count >= mount->dirs_capacity ) + { + mount->dirs_capacity *= 2; + struct assetsys_internal_folder_t* new_dirs = (struct assetsys_internal_folder_t*) ASSETSYS_MALLOC( + sys->memctx, sizeof( *(mount->dirs) ) * mount->dirs_capacity ); + memcpy( new_dirs, mount->dirs, sizeof( *(mount->dirs) ) * mount->dirs_count ); + ASSETSYS_FREE( sys->memctx, mount->dirs ); + mount->dirs = new_dirs; + } +// OpenJazz modification +#ifdef OJ_RECURSE + struct assetsys_internal_folder_t* as_dir = &mount->dirs[ mount->dirs_count++ ]; + as_dir->collated_index = assetsys_internal_register_collated( sys, sys->temp, 0 ); + assetsys_internal_recurse_directories( sys, as_dir->collated_index, mount ); +#endif + } + } + } + assetsys_internal_dir_close( &dir ); + } + +/** + * Mount the given internal mount data into the assetsys instance. + * + * @internal + */ +assetsys_error_t assetsys_internal_mount_files( assetsys_t* sys, struct assetsys_internal_mount_t* mount ) + { + int count = (int) mz_zip_reader_get_num_files( &mount->zip ); + + for( int i = 0; i < count; ++i ) + { + if( mz_zip_reader_is_file_a_directory( &mount->zip, (mz_uint) i ) ) + { + if( mount->dirs_count >= mount->dirs_capacity ) + { + mount->dirs_capacity *= 2; + struct assetsys_internal_folder_t* new_dirs = (struct assetsys_internal_folder_t*) ASSETSYS_MALLOC( + sys->memctx, sizeof( *(mount->dirs) ) * mount->dirs_capacity ); + memcpy( new_dirs, mount->dirs, sizeof( *(mount->dirs) ) * mount->dirs_count ); + ASSETSYS_FREE( sys->memctx, mount->dirs ); + mount->dirs = new_dirs; + } + + char filename[ 1024 ]; + mz_zip_reader_get_filename( &mount->zip, (mz_uint) i, filename, sizeof( filename ) ); + + struct assetsys_internal_folder_t* as_dir = &mount->dirs[ mount->dirs_count++ ]; + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->mounted_as ) ); + strcat( sys->temp, "/" ); + strcat( sys->temp, filename ); + sys->temp[ strlen( sys->temp ) - 1 ] = '\0'; + as_dir->collated_index = assetsys_internal_register_collated( sys, sys->temp, 0 ); + } + } + + for( int i = 0; i < count; ++i ) + { + if( !mz_zip_reader_is_file_a_directory( &mount->zip, (mz_uint) i ) ) + { + if( mount->files_count >= mount->files_capacity ) + { + mount->files_capacity *= 2; + struct assetsys_internal_file_t* new_files = (struct assetsys_internal_file_t*) ASSETSYS_MALLOC( + sys->memctx, sizeof( *(mount->files) ) * mount->files_capacity ); + memcpy( new_files, mount->files, sizeof( *(mount->files) ) * mount->files_count ); + ASSETSYS_FREE( sys->memctx, mount->files ); + mount->files = new_files; + } + + mz_zip_archive_file_stat stat; + mz_bool result = mz_zip_reader_file_stat( &mount->zip, (mz_uint) i, &stat); + if( !result ) + { + mz_zip_reader_end( &mount->zip ); + ASSETSYS_FREE( sys->memctx, mount->dirs ); + ASSETSYS_FREE( sys->memctx, mount->files ); + return ASSETSYS_ERROR_FAILED_TO_READ_ZIP; + } + + struct assetsys_internal_file_t* file = &mount->files[ mount->files_count++ ]; + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->mounted_as ) ); + strcat( sys->temp, "/" ); + strcat( sys->temp, stat.m_filename ); + file->collated_index = assetsys_internal_register_collated( sys, sys->temp, 1 ); + file->size = (int) stat.m_uncomp_size; + file->zip_index = i; + + char* dir_path = assetsys_internal_dirname( sys->temp ); + ASSETSYS_U64 handle = strpool_inject( &sys->strpool, dir_path, (int) strlen( dir_path ) - 1 ); + int found = 0; + for( int j = 0; j < mount->dirs_count; ++j ) + { + if( handle == sys->collated[ mount->dirs[ j ].collated_index ].path ) + found = 1; + } + if( !found ) + { + struct assetsys_internal_folder_t* as_dir = &mount->dirs[ mount->dirs_count++ ]; + as_dir->collated_index = assetsys_internal_register_collated( sys, + assetsys_internal_get_string( sys, handle ), 0 ); + } + } + } + + return ASSETSYS_SUCCESS; + } + +/** + * Creates an internal mount object for use by assetsys. + * + * @internal + */ +struct assetsys_internal_mount_t* assetsys_internal_create_mount( assetsys_t* sys, enum assetsys_internal_mount_type_t type, char const* path, char const* mount_as ) + { + if( sys->mounts_count >= sys->mounts_capacity ) + { + sys->mounts_capacity *= 2; + struct assetsys_internal_mount_t* new_mounts = (struct assetsys_internal_mount_t*) ASSETSYS_MALLOC( sys->memctx, + sizeof( *sys->mounts ) * sys->mounts_capacity ); + memcpy( new_mounts, sys->mounts, sizeof( *sys->mounts ) * sys->mounts_count ); + ASSETSYS_FREE( sys->memctx, sys->mounts ); + sys->mounts = new_mounts; + } + + struct assetsys_internal_mount_t* mount = &sys->mounts[ sys->mounts_count ]; + + mount->mounted_as = assetsys_internal_add_string( sys, mount_as ? mount_as : "" ); + mount->mount_len = mount_as ? (int) strlen( mount_as ) : 0; + mount->path = assetsys_internal_add_string( sys, path ); + mount->type = type; + + mount->files_count = 0; + mount->files_capacity = 4096; + mount->files = (struct assetsys_internal_file_t*) ASSETSYS_MALLOC( sys->memctx, + sizeof( *(mount->files) ) * mount->files_capacity ); + + mount->dirs_count = 0; + mount->dirs_capacity = 1024; + mount->dirs = (struct assetsys_internal_folder_t*) ASSETSYS_MALLOC( sys->memctx, + sizeof( *(mount->dirs) ) * mount->dirs_capacity ); + + if( type == ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP ) + { + memset( &mount->zip, 0, sizeof( mount->zip ) ); + mount->zip.m_pAlloc = assetsys_internal_mz_alloc; + mount->zip.m_pRealloc = assetsys_internal_mz_realloc; + mount->zip.m_pFree = assetsys_internal_mz_free; + mount->zip.m_pAlloc_opaque = sys->memctx; + } + + return mount; + } + +assetsys_error_t assetsys_mount_from_memory( assetsys_t* sys, void const* data, int size, char const* mount_as) + { + if (!data) return ASSETSYS_ERROR_INVALID_PARAMETER; + if( !mount_as ) return ASSETSYS_ERROR_INVALID_PARAMETER; + if( strchr( mount_as, '\\' ) ) return ASSETSYS_ERROR_INVALID_PATH; + int mount_len = (int) strlen( mount_as ); + if( mount_len == 0 || mount_as[ 0 ] != '/' || ( mount_len > 1 && mount_as[ mount_len - 1 ] == '/' ) ) + return ASSETSYS_ERROR_INVALID_PATH; + + struct assetsys_internal_mount_t* mount = assetsys_internal_create_mount(sys, ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP, "data", mount_as); + + mz_bool status = mz_zip_reader_init_mem( &mount->zip, data, size, 0 ); + if( !status ) + { + ASSETSYS_FREE( sys->memctx, mount->dirs ); + ASSETSYS_FREE( sys->memctx, mount->files ); + return ASSETSYS_ERROR_FAILED_TO_READ_ZIP; + } + + assetsys_error_t result = assetsys_internal_mount_files( sys, mount ); + if( result != ASSETSYS_SUCCESS ) + return result; + + assetsys_internal_collate_directories( sys, mount ); + + ++sys->mounts_count; + return ASSETSYS_SUCCESS; + } + +assetsys_error_t assetsys_mount( assetsys_t* sys, char const* path, char const* mount_as ) + { + if( !path ) return ASSETSYS_ERROR_INVALID_PARAMETER; + if( !mount_as ) return ASSETSYS_ERROR_INVALID_PARAMETER; + if( strchr( path, '\\' ) ) return ASSETSYS_ERROR_INVALID_PATH; + if( strchr( mount_as, '\\' ) ) return ASSETSYS_ERROR_INVALID_PATH; + int len = (int) strlen( path ); + if( len > 1 && path[ len - 1 ] == '/' ) return ASSETSYS_ERROR_INVALID_PATH; + int mount_len = (int) strlen( mount_as ); + if( mount_len == 0 || mount_as[ 0 ] != '/' || ( mount_len > 1 && mount_as[ mount_len - 1 ] == '/' ) ) + return ASSETSYS_ERROR_INVALID_PATH; + + enum assetsys_internal_mount_type_t type; + + #if defined( _MSC_VER ) && _MSC_VER >= 1400 + struct _stat64 s; + int res = __stat64( *path == '\0' ? "." : path, &s ); + #else + struct stat s; + int res = stat( *path == '\0' ? "." : path, &s ); + #endif + if( res == 0 ) + { + if( s.st_mode & S_IFDIR ) type = ASSETSYS_INTERNAL_MOUNT_TYPE_DIR; + else if( s.st_mode & S_IFREG ) type = ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP; + else return ASSETSYS_ERROR_INVALID_PATH; + } + else + { + return ASSETSYS_ERROR_INVALID_PATH; + } + + struct assetsys_internal_mount_t* mount = assetsys_internal_create_mount(sys, type, path, mount_as); + + if( type == ASSETSYS_INTERNAL_MOUNT_TYPE_DIR ) + { + struct assetsys_internal_folder_t* dir = &mount->dirs[ mount->dirs_count++ ]; + dir->collated_index = assetsys_internal_register_collated( sys, mount_as, 0 ); + assetsys_internal_recurse_directories( sys, dir->collated_index, mount ); + } + else if( type == ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP ) + { +#ifdef MINIZ_NO_STDIO + // If we explicitly disable stdio. + return ASSETSYS_ERROR_FAILED_TO_READ_ZIP; +#else + mz_bool status = mz_zip_reader_init_file( &mount->zip, path, 0 ); + if( !status ) + { + ASSETSYS_FREE( sys->memctx, mount->dirs ); + ASSETSYS_FREE( sys->memctx, mount->files ); + return ASSETSYS_ERROR_FAILED_TO_READ_ZIP; + } + + assetsys_error_t result = assetsys_internal_mount_files( sys, mount ); + if( result != ASSETSYS_SUCCESS ) + return result; +#endif + } + + assetsys_internal_collate_directories( sys, mount ); + + ++sys->mounts_count; + return ASSETSYS_SUCCESS; + } + + +static void assetsys_internal_remove_collated( assetsys_t* sys, int const index ) + { + struct assetsys_internal_collated_t* coll = &sys->collated[ index ]; + ASSETSYS_ASSERT( coll->ref_count > 0, "Invalid ref count" ); + --coll->ref_count; + if( coll->ref_count == 0 ) + { + strpool_decref( &sys->strpool, coll->path ); + strpool_discard( &sys->strpool, coll->path ); + } + } + + +assetsys_error_t assetsys_dismount( assetsys_t* sys, char const* path, char const* mounted_as ) + { + if( !path ) return ASSETSYS_ERROR_INVALID_PARAMETER; + if( !mounted_as ) return ASSETSYS_ERROR_INVALID_MOUNT; + + ASSETSYS_U64 path_handle = strpool_inject( &sys->strpool, path, (int) strlen( path ) ); + ASSETSYS_U64 mount_handle = strpool_inject( &sys->strpool, mounted_as, (int) strlen( mounted_as ) ); + + for( int i = 0; i < sys->mounts_count; ++i ) + { + struct assetsys_internal_mount_t* mount = &sys->mounts[ i ]; + if( mount->mounted_as == mount_handle && mount->path == path_handle ) + { + mz_bool result = 1; + if( mount->type == ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP ) result = mz_zip_reader_end( &mount->zip ); + + strpool_decref( &sys->strpool, mount->mounted_as ); + strpool_decref( &sys->strpool, mount->path ); + strpool_discard( &sys->strpool, mount_handle ); + strpool_discard( &sys->strpool, path_handle ); + + for( int j = 0; j < mount->dirs_count; ++j ) + assetsys_internal_remove_collated( sys, mount->dirs[ j ].collated_index ); + + for( int j = 0; j < mount->files_count; ++j ) + assetsys_internal_remove_collated( sys, mount->files[ j ].collated_index ); + + ASSETSYS_FREE( sys->memctx, mount->dirs ); + ASSETSYS_FREE( sys->memctx, mount->files ); + + int count = sys->mounts_count - i; + if( count > 0 ) memcpy( &sys->mounts[ i ], &sys->mounts[ i + 1 ], sizeof( *sys->mounts ) * count ); + --sys->mounts_count; + + return !result ? ASSETSYS_ERROR_FAILED_TO_CLOSE_ZIP : ASSETSYS_SUCCESS; + } + } + + strpool_discard( &sys->strpool, mount_handle ); + strpool_discard( &sys->strpool, path_handle ); + return ASSETSYS_ERROR_INVALID_MOUNT; + } + + +assetsys_error_t assetsys_file( assetsys_t* sys, char const* path, assetsys_file_t* file ) + { + if( !file || !path ) return ASSETSYS_ERROR_INVALID_PARAMETER; + + ASSETSYS_U64 handle = strpool_inject( &sys->strpool, path, (int) strlen( path ) ); + + int m = sys->mounts_count; + while( m > 0) + { + --m; + struct assetsys_internal_mount_t* mount = &sys->mounts[ m ]; + for( int i = 0; i < mount->files_count; ++i ) + { + ASSETSYS_U64 h = sys->collated[ mount->files[ i ].collated_index ].path; + if( handle == h ) + { + file->mount = mount->mounted_as; + file->path = mount->path; + file->index = i; + return ASSETSYS_SUCCESS; + } + } + } + + strpool_discard( &sys->strpool, handle ); + return ASSETSYS_ERROR_FILE_NOT_FOUND; + } + + +static int assetsys_internal_find_mount_index( assetsys_t* sys, ASSETSYS_U64 const mount, ASSETSYS_U64 const path ) + { + for( int i = 0; i < sys->mounts_count; ++i ) + { + if( sys->mounts[ i ].mounted_as == mount && sys->mounts[ i ].path == path ) + return i; + } + return -1; + } + + +assetsys_error_t assetsys_file_load( assetsys_t* sys, assetsys_file_t f, int* size, void* buffer, int capacity ) + { + int mount_index = assetsys_internal_find_mount_index( sys, f.mount, f.path ); + if( mount_index < 0 ) return ASSETSYS_ERROR_INVALID_MOUNT; + + struct assetsys_internal_mount_t* mount = &sys->mounts[ mount_index ]; + struct assetsys_internal_file_t* file = &mount->files[ f.index ]; + if( mount->type == ASSETSYS_INTERNAL_MOUNT_TYPE_ZIP ) + { + if( size ) *size = (int) file->size; + if( file->size > capacity ) return ASSETSYS_ERROR_BUFFER_TOO_SMALL; + + mz_bool result = mz_zip_reader_extract_to_mem_no_alloc( &mount->zip, (mz_uint) file->zip_index, buffer, + (size_t) file->size, 0, 0, 0 ); + return result ? ASSETSYS_SUCCESS : ASSETSYS_ERROR_FAILED_TO_READ_FILE; + } + else + { + if( size ) *size = file->size; + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->path ) ); + strcat( sys->temp, *sys->temp == '\0' ? "" : "/" ); + strcat( sys->temp, assetsys_internal_get_string( sys, + sys->collated[ file->collated_index ].path ) + + ( strcmp( assetsys_internal_get_string( sys, mount->mounted_as ), "/" ) == 0 ? 0 : mount->mount_len + 1 ) ); + FILE* fp = fopen( sys->temp, "rb" ); + if( !fp ) return ASSETSYS_ERROR_FAILED_TO_READ_FILE; + + fseek( fp, 0, SEEK_END ); + int file_size = (int) ftell( fp ); + fseek( fp, 0, SEEK_SET ); + if( size ) *size = file_size; + + if( file_size > capacity ) { fclose( fp ); return ASSETSYS_ERROR_BUFFER_TOO_SMALL; } + + int size_read = (int) fread( buffer, 1, (size_t) file_size, fp ); + fclose( fp ); + if( size_read != file_size ) return ASSETSYS_ERROR_FAILED_TO_READ_FILE; + + return ASSETSYS_SUCCESS; + } + } + + +int assetsys_file_size( assetsys_t* sys, assetsys_file_t file ) + { + int mount_index = assetsys_internal_find_mount_index( sys, file.mount, file.path ); + if( mount_index < 0 ) return 0; + + struct assetsys_internal_mount_t* mount = &sys->mounts[ mount_index ]; + if( mount->type == ASSETSYS_INTERNAL_MOUNT_TYPE_DIR ) + { + strcpy( sys->temp, assetsys_internal_get_string( sys, mount->path ) ); + strcat( sys->temp, *sys->temp == '\0' ? "" : "/" ); + strcat( sys->temp, assetsys_internal_get_string( sys, + sys->collated[ mount->files[ file.index ].collated_index ].path ) + mount->mount_len + 1 ); + struct stat s; + if( stat( sys->temp, &s ) == 0 ) + mount->files[ file.index ].size = (int) s.st_size; + } + + return mount->files[ file.index ].size; + } + + +static int assetsys_internal_find_collated( assetsys_t* sys, char const* const path ) + { + ASSETSYS_U64 handle = strpool_inject( &sys->strpool, path, (int) strlen( path ) ); + + for( int i = 0; i < sys->collated_count; ++i ) + { + if( sys->collated[ i ].path == handle ) + { + return i; + } + } + + strpool_discard( &sys->strpool, handle ); + return -1; + } + + +int assetsys_file_count( assetsys_t* sys, char const* path ) + { + if( !path ) return 0; + int dir = assetsys_internal_find_collated( sys, path ); + int count = 0; + for( int i = 0; i < sys->collated_count; ++i ) + { + if( sys->collated[ i ].is_file && sys->collated[ i ].parent == dir ) + { + ++count; + } + } + return count; + } + + +char const* assetsys_file_name( assetsys_t* sys, char const* path, int index ) + { + char const* file_path = assetsys_file_path( sys, path, index ); + if( file_path ) + { + char const* name = strrchr( file_path, '/' ); + if( !name ) return file_path; + return name + 1; + } + + return NULL; + } + + +char const* assetsys_file_path( assetsys_t* sys, char const* path, int index ) + { + if( !path ) return 0; + int dir = assetsys_internal_find_collated( sys, path ); + int count = 0; + for( int i = 0; i < sys->collated_count; ++i ) + { + if( sys->collated[ i ].is_file && sys->collated[ i ].parent == dir ) + { + if( count == index ) return assetsys_internal_get_string( sys, sys->collated[ i ].path ); + ++count; + } + } + return NULL; + } + + +int assetsys_subdir_count( assetsys_t* sys, char const* path ) + { + if( !path ) return 0; + int dir = assetsys_internal_find_collated( sys, path ); + int count = 0; + for( int i = 0; i < sys->collated_count; ++i ) + { + if( !sys->collated[ i ].is_file && sys->collated[ i ].parent == dir ) + { + ++count; + } + } + return count; + } + + +char const* assetsys_subdir_name( assetsys_t* sys, char const* path, int index ) + { + char const* subdir_path = assetsys_subdir_path( sys, path, index ); + if( subdir_path ) + { + char const* name = strrchr( subdir_path, '/' ); + if( !name ) return subdir_path; + return name + 1; + } + + return NULL; + } + + +char const* assetsys_subdir_path( assetsys_t* sys, char const* path, int index ) + { + if( !path ) return 0; + int dir = assetsys_internal_find_collated( sys, path ); + int count = 0; + for( int i = 0; i < sys->collated_count; ++i ) + { + if( !sys->collated[ i ].is_file && sys->collated[ i ].parent == dir ) + { + if( count == index ) return assetsys_internal_get_string( sys, sys->collated[ i ].path ); + ++count; + } + } + return NULL; + } + + +const char *assetsys_file_to_path(assetsys_t* sys, assetsys_file_t file) { + int mount_index = assetsys_internal_find_mount_index(sys, file.mount, file.path); + if(mount_index < 0) return ""; + + sys->temp[0] = '\0'; + + struct assetsys_internal_mount_t* mount = &sys->mounts[mount_index]; + strcpy(sys->temp, assetsys_internal_get_string(sys, mount->path)); + strcat(sys->temp, *sys->temp == '\0' ? "" : "/" ); + strcat(sys->temp, assetsys_internal_get_string(sys, + sys->collated[mount->files[file.index].collated_index].path) + mount->mount_len + 1); + + return sys->temp; +} + +static char* assetsys_internal_dirname( char const* path ) + { + static char result[ 260 ]; + strncpy( result, path, sizeof( result ) ); + + char* lastForwardSlash = strrchr( result, '/' ); + + if( lastForwardSlash ) *(lastForwardSlash + 1 ) = '\0'; + else *result = '\0'; + + return result; + } + + +#endif /* ASSETSYS_IMPLEMENTATION */ + + +/* +---------------------- + TESTS +---------------------- +*/ + + +#ifdef ASSETSYS_RUN_TESTS + +#include "testfw.h" + + +void test_assetsys( void ) { + TESTFW_TEST_BEGIN( "Test mounting path and loading file" ); + + // Create the asset system + assetsys_t* assetsys = assetsys_create( 0 ); + TESTFW_EXPECTED( assetsys != NULL ); + + // Mount current working folder as a virtual "/data" path + TESTFW_EXPECTED( assetsys_mount( assetsys, ".", "/data" ) == ASSETSYS_SUCCESS ); + + // Load a file + assetsys_file_t file; + TESTFW_EXPECTED( assetsys_file( assetsys, "/data/README.md", &file ) == ASSETSYS_SUCCESS ); + + // Find the size of the file + int size = assetsys_file_size( assetsys, file ); + TESTFW_EXPECTED( size > 30 ); + + // Load the file + char* content = (char*) malloc( size + 1 ); // extra space for '\0' + int loaded_size = 0; + TESTFW_EXPECTED( assetsys_file_load( assetsys, file, &loaded_size, content, size ) == ASSETSYS_SUCCESS ); + content[ size ] = '\0'; // zero terminate the text file + + // Clean up + free( content ); + assetsys_destroy( assetsys ); + + TESTFW_TEST_END(); + + TESTFW_TEST_BEGIN( "Test mounting data and loading file" ); + { + // Zip file with a test.txt file containing "Hello, World!" + const unsigned char data[] = { + 0x50, 0x4b, 0x03, 0x04, 0x0a, 0x03, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x9e, 0x23, 0x57, 0x84, 0x9e, + 0xe8, 0xb4, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x74, 0x65, + 0x73, 0x74, 0x2e, 0x74, 0x78, 0x74, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, + 0x6c, 0x64, 0x21, 0x0a, 0x50, 0x4b, 0x01, 0x02, 0x3f, 0x03, 0x0a, 0x03, 0x00, 0x00, 0x00, 0x00, + 0xfc, 0x9e, 0x23, 0x57, 0x84, 0x9e, 0xe8, 0xb4, 0x0e, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x80, 0xb4, 0x81, 0x00, 0x00, + 0x00, 0x00, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x74, 0x78, 0x74, 0x0a, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x18, 0x00, 0x00, 0x0e, 0xf2, 0x2d, 0xc2, 0xde, 0xd9, 0x01, 0x00, 0x0e, + 0xf2, 0x2d, 0xc2, 0xde, 0xd9, 0x01, 0x00, 0x0e, 0xf2, 0x2d, 0xc2, 0xde, 0xd9, 0x01, 0x50, 0x4b, + 0x05, 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x5a, 0x00, 0x00, 0x00, 0x34, 0x00, + 0x00, 0x00, 0x00, 0x00 + }; + const int data_size = 164; + + // Create the asset system + assetsys_t* assetsys = assetsys_create( 0 ); + TESTFW_EXPECTED( assetsys != NULL ); + + // Mount current working folder as a virtual "/data" path + TESTFW_EXPECTED( assetsys_mount_from_memory( assetsys, data, data_size, "/data" ) == ASSETSYS_SUCCESS ); + + // Load a file + assetsys_file_t file; + TESTFW_EXPECTED( assetsys_file( assetsys, "/data/test.txt", &file ) == ASSETSYS_SUCCESS ); + + // Find the size of the file + int size = assetsys_file_size( assetsys, file ); + TESTFW_EXPECTED( size > 10 ); + + // Load the file + char* content = (char*) malloc( size + 1 ); // extra space for '\0' + int loaded_size = 0; + TESTFW_EXPECTED( assetsys_file_load( assetsys, file, &loaded_size, content, size ) == ASSETSYS_SUCCESS ); + content[ size ] = '\0'; // zero terminate the text file + + TESTFW_EXPECTED( content[0] == 'H' ); + TESTFW_EXPECTED( content[1] == 'e' ); + TESTFW_EXPECTED( content[2] == 'l' ); + TESTFW_EXPECTED( content[3] == 'l' ); + TESTFW_EXPECTED( content[4] == 'o' ); + + // Clean up + free( content ); + assetsys_destroy( assetsys ); + } + TESTFW_TEST_END(); +} + + +int main( int argc, char** argv ) { + (void) argc, argv; + + TESTFW_INIT(); + + test_assetsys(); + + return TESTFW_SUMMARY(); +} + + +// pass-through so the program will build with either /SUBSYSTEM:WINDOWS or /SUBSYSTEM:CONSOLE +#if defined( _WIN32 ) && !defined( __TINYC__ ) + #include + #ifdef __cplusplus + extern "C" int __stdcall WinMain( struct HINSTANCE__*, struct HINSTANCE__*, char*, int ) { + return main( __argc, __argv ); + } + #else + struct HINSTANCE__; + int __stdcall WinMain( struct HINSTANCE__* a, struct HINSTANCE__* b, char* c, int d ) { + (void) a, b, c, d; return main( __argc, __argv ); + } + #endif +#endif + +#define TESTFW_IMPLEMENTATION +#include "testfw.h" + +#endif /* ASSETSYS_RUN_TESTS */ + + +/* + +contributors: + Randy Gaul (hotloading support) + Rob Loach (assetsys_mount_from_memory) + +revision history: + 1.5 fix issue where mount as root "/" didn't work when mounting folder + 1.4 allow mounting from memory + 1.3 allow absolute paths when mounting, update docs for mount as root + 1.2 asserts with message, eliminated a frequent small allocation + 1.1 changes to support loading assets being re-saved during execution + 1.0 first released version + +*/ + + +/* +------------------------------------------------------------------------------ + +This software is available under 2 licenses - you may choose the one you like. +Uses public domain code for "miniz" zip file support - original license can be +found in the miniz section of this file. + +------------------------------------------------------------------------------ + +ALTERNATIVE A - MIT License + +Copyright (c) 2015 Mattias Gustavsson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------ + +ALTERNATIVE B - Public Domain (www.unlicense.org) + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------ +*/ + diff --git a/ext/mg/strpool.cpp b/ext/mg/strpool.cpp new file mode 100644 index 00000000..b29f3c0b --- /dev/null +++ b/ext/mg/strpool.cpp @@ -0,0 +1,2 @@ +#define STRPOOL_IMPLEMENTATION +#include "strpool.h" diff --git a/ext/mg/strpool.h b/ext/mg/strpool.h new file mode 100644 index 00000000..38a6e801 --- /dev/null +++ b/ext/mg/strpool.h @@ -0,0 +1,1313 @@ +/* +------------------------------------------------------------------------------ + Licensing information can be found at the end of the file. +------------------------------------------------------------------------------ + +strpool.h - v1.4 - Highly efficient string pool for C/C++. + +Do this: + #define STRPOOL_IMPLEMENTATION +before you include this file in *one* C/C++ file to create the implementation. +*/ + +#ifndef strpool_h +#define strpool_h + +// OpenJazz Modifications +#include +#define STRPOOL_U32 uint32_t +#define STRPOOL_U64 uint64_t +// End of Openjazz Modifications + +#ifndef STRPOOL_U32 + #define STRPOOL_U32 unsigned int +#endif +#ifndef STRPOOL_U64 + #define STRPOOL_U64 unsigned long long +#endif + +typedef struct strpool_t strpool_t; + +typedef struct strpool_config_t + { + void* memctx; + int ignore_case; + int counter_bits; + int index_bits; + int entry_capacity; + int block_capacity; + int block_size; + int min_length; + } strpool_config_t; + +extern strpool_config_t const strpool_default_config; + +void strpool_init( strpool_t* pool, strpool_config_t const* config ); +void strpool_term( strpool_t* pool ); + +void strpool_defrag( strpool_t* pool ); + +STRPOOL_U64 strpool_inject( strpool_t* pool, char const* string, int length ); +void strpool_discard( strpool_t* pool, STRPOOL_U64 handle ); + +int strpool_incref( strpool_t* pool, STRPOOL_U64 handle ); +int strpool_decref( strpool_t* pool, STRPOOL_U64 handle ); +int strpool_getref( strpool_t* pool, STRPOOL_U64 handle ); + +int strpool_isvalid( strpool_t const* pool, STRPOOL_U64 handle ); + +char const* strpool_cstr( strpool_t const* pool, STRPOOL_U64 handle ); +int strpool_length( strpool_t const* pool, STRPOOL_U64 handle ); + +char* strpool_collate( strpool_t const* pool, int* count ); +void strpool_free_collated( strpool_t const* pool, char* collated_ptr ); + +#endif /* strpool_h */ + + +/** +strpool.h +========= + +Highly efficient string pool for C/C++. + + +Example +------- + + #define STRPOOL_IMPLEMENTATION + #include "strpool.h" + + #include // for printf + #include // for strlen + + int main( int argc, char** argv ) { + + strpool_config_t conf = strpool_default_config; + //conf.ignore_case = true; + + strpool_t pool; + strpool_init( &pool, &conf ); + + STRPOOL_U64 str_a = strpool_inject( &pool, "This is a test string", (int) strlen( "This is a test string" ) ); + STRPOOL_U64 str_b = strpool_inject( &pool, "THIS IS A TEST STRING", (int) strlen( "THIS IS A TEST STRING" ) ); + + printf( "%s\n", strpool_cstr( &pool, str_a ) ); + printf( "%s\n", strpool_cstr( &pool, str_b ) ); + printf( "%s\n", str_a == str_b ? "Strings are the same" : "Strings are different" ); + + strpool_term( &pool ); + return 0; + } + + +API Documentation +----------------- + +strpool.h is a system for string interning, where each string is stored only once. It makes comparing strings very fast, +as it will just be uint64 comparison rather than looping over strings and comparing character by character. strpool.h is +also optimized for fast creation of strings, and with an efficient memory allocation scheme. + +strpool.h is a single-header library, and does not need any .lib files or other binaries, or any build scripts. To use +it, you just include strpool.h to get the API declarations. To get the definitions, you must include strpool.h from +*one* single C or C++ file, and #define the symbol `STRPOOL_IMPLEMENTATION` before you do. + + +### Customization + +There are a few different things in strpool.h which are configurable by #defines. Most of the API use the `int` data +type, for integer values where the exact size is not important. However, for some functions, it specifically makes use +of 32 and 64 bit data types. These default to using `unsigned int` and `unsigned long long` by default, but can be +redefined by #defining STRPOOL_U32 and STRPOOL_U64 respectively, before including strpool.h. This is useful if you, for +example, use the types from `` in the rest of your program, and you want strpool.h to use compatible types. In +this case, you would include strpool.h using the following code: + + #define STRPOOL_U32 uint32_t + #define STRPOOL_U64 uint64_t + #include "strpool.h" + +Note that when customizing the data types, you need to use the same definition in every place where you include +strpool.h, as they affect the declarations as well as the definitions. + +The rest of the customizations only affect the implementation, so will only need to be defined in the file where you +have the #define STRPOOL_IMPLEMENTATION. + +Note that if all customizations are utilized, strpool.h will include no external files whatsoever, which might be useful +if you need full control over what code is being built. + + +#### Custom memory allocators + +To store strings and the internal structures (entry list, hashtable etc) strpool.h needs to dodynamic allocation by +calling `malloc`. Programs might want to keep track of allocations done, or use custom defined pools to allocate memory +from. strpool.h allows for specifying custom memory allocation functions for `malloc` and `free`. +This is done with the following code: + + #define STRPOOL_IMPLEMENTATION + #define STRPOOL_MALLOC( ctx, size ) ( my_custom_malloc( ctx, size ) ) + #define STRPOOL_FREE( ctx, ptr ) ( my_custom_free( ctx, ptr ) ) + #include "strpool.h" + +where `my_custom_malloc` and `my_custom_free` are your own memory allocation/deallocation functions. The `ctx` parameter +is an optional parameter of type `void*`. When `strpool_init` is called, you can set the `memctx` field of the `config` +parameter, to a pointer to anything you like, and which will be passed through as the `ctx` parameter to every +`STRPOOL_MALLOC`/`STRPOOL_FREE` call. For example, if you are doing memory tracking, you can pass a pointer to your +tracking data as `memctx`, and in your custom allocation/deallocation function, you can cast the `ctx` param back to the +right type, and access the tracking data. + +If no custom allocator is defined, strpool.h will default to `malloc` and `free` from the C runtime library. + + +#### Custom assert + +strpool.h makes use of asserts to report usage errors and failed allocation errors. By default, it makes use of the C +runtime library `assert` macro, which only executes in debug builds. However, it allows for substituting with your own +assert function or macro using the following code: + + #define STRPOOL_IMPLEMENTATION + #define STRPOOL_ASSERT( condition ) ( my_custom_assert( condition ) ) + #include "strpool.h" + +Note that if you only want the asserts to trigger in debug builds, you must add a check for this in your custom assert. + + +#### Custom C runtime function + +The library makes use of four additional functions from the C runtime library, and for full flexibility, it allows you +to substitute them for your own. Here's an example: + + #define STRPOOL_IMPLEMENTATION + #define STRPOOL_MEMSET( ptr, val, cnt ) ( my_memset_func( ptr, val, cnt ) ) + #define STRPOOL_MEMCPY( dst, src, cnt ) ( my_memcpy_func( dst, src, cnt ) ) + #define STRPOOL_MEMCMP( pr1, pr2, cnt ) ( my_memcmp_func( pr1, pr2, cnt ) ) + #define STRPOOL_STRNICMP( s1, s2, len ) ( my_strnicmp_func( s1, s2, len ) ) + #include "strpool.h" + +If no custom function is defined, strpool.h will default to the C runtime library equivalent. + + +strpool_init +------------ + + void strpool_init( strpool_t* pool, strpool_config_t const* config ) + +Initializes an instance of the string pool. The `config` parameter allows for controlling the behavior of the pool +instance. It contains the following fields: + +* memctx - pointer to user defined data which will be passed through to custom STRPOOL_MALLOC/STRPOOL_FREE calls. May + be NULL. +* ignore_case - set to 0 to make strings case sensitive, set to 1 to make strings case insensitive. Default is 0. +* counter_bits - how many bits of the string handle to use for keeping track of handle reuse and invalidation. Default + is 32. See below for details about the handle bits. +* index_bits - how many bits of the string handle to use for referencing string instances. Default is 32. See below for + details about the handle bits. +* entry_capacity - number of string instance entries to pre-allocate space for when pool is initialized. Default + is space for 4096 string entries. +* block_capacity - number of string storage block entries to pre-allocate space for when pool is initialized. Default + is space for 32 entries of block information - though only a single block will be pre-allocated. +* block_size - size to allocate for each string storage block. A higher value might mean you often have more memory + allocated than is actually being used. A lower value means you will be making allocations more often. Default + is 256 kilobyte. +* min_length - minimum space to allocate for each string. A higher value wastes more space, but makes it more likely + that recycled storage can be re-used by subsequent requests. Default is a string length of 23 characters. + +The function `strpool_inject` returns a 64-bit handle. Using the settings `counter_bits`/`index_bits`, you can control +how many bits of the handle is in use, and how many are used for index vs counter. For example, setting `counter_bits` +to 8 and `index_bits` to 24, you will get a handle which only uses 32 bits in total, can store some 16 million different +strings (2^24 -1), with a reuse counter going up to 255 before wrapping around. In this scenario, the 64-bit handle +returned from strpool_inject can be safely truncated and stored in a 32-bit variable. Any combination of counter vs +index bits is possible, but the number of strings which can be stored will be limited by `index_bits`. + +The counter bits need a more detailed description. When a string is added via `strpool_inject`, you get a handle back, +which is used to reference the string. You might store this handle in various data structures. When removing a string by +calling `strpool_discard`, the index part of that handle will be recycled, and re-used for future `strpool_inject`. +However, when a string is discarded, the counter part of the handle will be increased, so the system knows that any +handles that are still being kept around in any data structure, does no longer point to a valid string (and thus will +return a NULL string pointer when queried through `strpool_cstr`). If your use case involves creating a bunch of strings +and just keeping them around until the application terminates, it is fine to specify 0 for `counter_bits`, thereby +effectively disabling the handle validation all together. If you have a case where strings are being repeatedly created +and removed, but strings are queried very frequently, then you can specify a low number for `counter_bits` (since you +can invalidate any stored handles as soon as you find out it's been invalidated). If you have a case where handles might +sit inside data structures for a long period of time before you check their validity, you best specify a high value for +`counter_bits`. The default value is 32 bits for index and 32 bits for counter. + + +strpool_term +------------ + + void strpool_term( strpool_t* pool ) + +Terminates a string pool instance, releasing all memory used by it. No further calls to the strpool API are valid until +the instance is reinitialized by another call to `strpool_init`. + + +strpool_defrag +-------------- + + void strpool_defrag( strpool_t* pool ) + +As strings are added to and removed from the pool, the memory blocks holding the strings may get fragmented, making them +take up more allocated memory than is actually needed to store the set of strings, should they be packed tightly, +`strpool_defrag` consolidates all strings into a single memory block (which might well be larger than the block size +specified when the pool was initialized), and recreates the internal hash table. Any additionally allocated memory +blocks will be deallocated, making the memory used by the pool as little as it can be to fit the current set of strings. +All string handles remain valid after a call to `strpool_defrag`. + + +strpool_inject +-------------- + + STRPOOL_U64 strpool_inject( strpool_t* pool, char const* string, int length ) + +Adds a string to the pool, and returns a handle for the added string. If the string doesn't exist, it will be stored in +an unused part of an already allocated storage block (or a new block will be allocated if there is no free space large +enough to hold the string), and inserted into the internal hash table. If the string already exists, a handle to the +existing string will be returned. If the `string` parameter is already pointing to a string stored in the pool, there +are specific optimizations for avoiding to loop over it to calculate the hash for it, with the idea being that you +should be able to use char* types when you're passing strings around, and use the string pool for storage, without too +much of a performance penalty. `string` does not have to be null terminated, but when it is retrieved from the string +pool, it will be. If `string` is NULL or length is 0, a handle with a value of 0 will be returned, to signify empty +string. + + +strpool_discard +--------------- + + void strpool_discard( strpool_t* pool, STRPOOL_U64 handle ) + +Removes a string from the pool. Any handles held for the string will be invalid after this. Memory used for storing the +string will be recycled and used for further `strpool_inject` calls. If `handle` is invalid, `strpool_discard` will do +nothing. + + +strpool_incref +-------------- + + int strpool_incref( strpool_t* pool, STRPOOL_U64 handle ) + +`strpool.h` supports reference counting of strings. It is optional and not automatic, and does not automatically discard +strings when the reference count reaches 0. To use reference counting, make sure to call `strpool_incref` whenever you +add a string (after having called `strpool_inject`), and to call `strpool_decref` whenever you want to remove a string +(but only call `strpool_discard` if reference count have reached 0). It would be advisable to write wrapper functions +to ensure consistency in this, or if C++ is used, a wrapper class with constructors/destructor. `strpool_incref` returns +the reference count after increasing it. If `handle` is invalid, `strpool_incref` will do nothing, and return 0. + + +strpool_decref +-------------- + + int strpool_decref( strpool_t* pool, STRPOOL_U64 handle ) + +Decreases the reference count of the specified string by 1, returning the reference count after decrementing. If the +reference count is less than 1, an assert will be triggered. If `handle` is invalid, `strpool_decref` will do nothing, +and return 0. + + +strpool_getref +-------------- + + int strpool_getref( strpool_t* pool, STRPOOL_U64 handle ) + +Returns the current reference count for the specified string. If `handle` is invalid, `strpool_getref` will do nothing, +and return 0. + + +strpool_isvalid +--------------- + + int strpool_isvalid( strpool_t const* pool, STRPOOL_U64 handle ) + +Returns 1 if the specified string handle is valid, and 0 if it is not. + + +strpool_cstr +------------ + + char const* strpool_cstr( strpool_t const* pool, STRPOOL_U64 handle ) + +Returns the zero-terminated C string for the specified string handle. The resulting string pointer is only valid as long +as no call is made to `strpool_init`, `strpool_term`, `strpool_defrag` or `strpool_discard`. It is therefor recommended +to never store the C string pointer, and always grab it fresh by another call to `strpool_cstr` when it is needed. +`strpool_cstr` is a very fast function to call - it does little more than an array lookup. If `handle` is invalid, +`strpool_cstr` returns NULL. + + +strpool_length +-------------- + + int strpool_length( strpool_t const* pool, STRPOOL_U64 handle ) + +Returns the length, in characters, of the specified string. The resulting value is only valid as long as no call is made +to `strpool_init`, `strpool_term`, `strpool_defrag` or `strpool_discard`. It is therefor recommended to never store the +value, and always grab it fresh by another call to `strpool_length` when it is needed. `strpool_length` is a very fast +function to call - it does little more than an array lookup. If `handle` is invalid, `strpool_length` returns 0. + + +strpool_collate +--------------- + + char* strpool_collate( strpool_t const* pool, int* count ) + +Returns a list of all the strings currently stored in the string pool, and stores the number of strings in the int +variable pointed to by `count`. If there are no strings in the string pool, `strpool_collate` returns NULL. The pointer +returned points to the first character of the first string. Strings are zero-terminated, and immediately after the +termination character, comes the first character of the next string. + + +strpool_free_collated +--------------------- + + void strpool_free_collated( strpool_t const* pool, char* collated_ptr ) + +Releases the memory returned by `strpool_collate`. + +*/ + + +/* +---------------------- + IMPLEMENTATION +---------------------- +*/ + +#ifndef strpool_impl +#define strpool_impl + +struct strpool_internal_hash_slot_t; +struct strpool_internal_entry_t; +struct strpool_internal_handle_t; +struct strpool_internal_block_t; + +struct strpool_t + { + void* memctx; + int ignore_case; + int counter_shift; + STRPOOL_U64 counter_mask; + STRPOOL_U64 index_mask; + + int initial_entry_capacity; + int initial_block_capacity; + int block_size; + int min_data_size; + + struct strpool_internal_hash_slot_t* hash_table; + int hash_capacity; + + struct strpool_internal_entry_t* entries; + int entry_capacity; + int entry_count; + + struct strpool_internal_handle_t* handles; + int handle_capacity; + int handle_count; + int handle_freelist_head; + int handle_freelist_tail; + + struct strpool_internal_block_t* blocks; + int block_capacity; + int block_count; + int current_block; + }; + + +#endif /* strpool_impl */ + + +#ifdef STRPOOL_IMPLEMENTATION +#undef STRPOOL_IMPLEMENTATION + +#define _CRT_NONSTDC_NO_DEPRECATE +#define _CRT_SECURE_NO_WARNINGS +#include + +#ifndef STRPOOL_ASSERT + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define STRPOOL_ASSERT( expression, message ) assert( ( expression ) && ( message ) ) +#endif + +#ifndef STRPOOL_MEMSET + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define STRPOOL_MEMSET( ptr, val, cnt ) ( memset( ptr, val, cnt ) ) +#endif + +#ifndef STRPOOL_MEMCPY + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define STRPOOL_MEMCPY( dst, src, cnt ) ( memcpy( dst, src, cnt ) ) +#endif + +#ifndef STRPOOL_MEMCMP + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define STRPOOL_MEMCMP( pr1, pr2, cnt ) ( memcmp( pr1, pr2, cnt ) ) +#endif + +#ifndef STRPOOL_STRNICMP + #ifdef _WIN32 + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define STRPOOL_STRNICMP( s1, s2, len ) ( strnicmp( s1, s2, len ) ) + #else + #include + #define STRPOOL_STRNICMP( s1, s2, len ) ( strncasecmp( s1, s2, len ) ) + #endif +#endif + +#ifndef STRPOOL_MALLOC + #define _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_SECURE_NO_WARNINGS + #include + #define STRPOOL_MALLOC( ctx, size ) ( malloc( size ) ) + #define STRPOOL_FREE( ctx, ptr ) ( free( ptr ) ) +#endif + + +typedef struct strpool_internal_hash_slot_t + { + STRPOOL_U32 hash_key; + int entry_index; + int base_count; + } strpool_internal_hash_slot_t; + + +typedef struct strpool_internal_entry_t + { + int hash_slot; + int handle_index; + char* data; + int size; + int length; + int refcount; + } strpool_internal_entry_t; + + +typedef struct strpool_internal_handle_t + { + int entry_index; + int counter; + } strpool_internal_handle_t; + + +typedef struct strpool_internal_block_t + { + int capacity; + char* data; + char* tail; + int free_list; + } strpool_internal_block_t; + + +typedef struct strpool_internal_free_block_t + { + int size; + int next; + } strpool_internal_free_block_t; + + +strpool_config_t const strpool_default_config = + { + /* memctx = */ 0, + /* ignore_case = */ 0, + /* counter_bits = */ 32, + /* index_bits = */ 32, + /* entry_capacity = */ 4096, + /* block_capacity = */ 32, + /* block_size = */ 256 * 1024, + /* min_length = */ 23, + }; + + + +static STRPOOL_U32 strpool_internal_pow2ceil( STRPOOL_U32 v ) + { + --v; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + ++v; + v += ( v == 0 ); + return v; + } + + +static int strpool_internal_add_block( strpool_t* pool, int size ) + { + if( pool->block_count >= pool->block_capacity ) + { + pool->block_capacity *= 2; + strpool_internal_block_t* new_blocks = (strpool_internal_block_t*) STRPOOL_MALLOC( pool->memctx, + pool->block_capacity * sizeof( *pool->blocks ) ); + STRPOOL_ASSERT( new_blocks, "Allocation failed" ); + STRPOOL_MEMCPY( new_blocks, pool->blocks, pool->block_count * sizeof( *pool->blocks ) ); + STRPOOL_FREE( pool->memctx, pool->blocks ); + pool->blocks = new_blocks; + } + pool->blocks[ pool->block_count ].capacity = size; + pool->blocks[ pool->block_count ].data = (char*) STRPOOL_MALLOC( pool->memctx, (size_t) size ); + STRPOOL_ASSERT( pool->blocks[ pool->block_count ].data, "Allocation failed" ); + pool->blocks[ pool->block_count ].tail = pool->blocks[ pool->block_count ].data; + pool->blocks[ pool->block_count ].free_list = -1; + return pool->block_count++; + } + + +void strpool_init( strpool_t* pool, strpool_config_t const* config ) + { + if( !config ) config = &strpool_default_config; + + pool->memctx = config->memctx; + pool->ignore_case = config->ignore_case; + + STRPOOL_ASSERT( config->counter_bits + config->index_bits <= 64, "Total bit count exceeds 64" ); + pool->counter_shift = config->index_bits; + pool->counter_mask = ( 1ULL << (STRPOOL_U64) config->counter_bits ) - 1; + pool->index_mask = ( 1ULL << (STRPOOL_U64) config->index_bits ) - 1; + + pool->initial_entry_capacity = + (int) strpool_internal_pow2ceil( config->entry_capacity > 1 ? (STRPOOL_U32)config->entry_capacity : 2U ); + pool->initial_block_capacity = + (int) strpool_internal_pow2ceil( config->block_capacity > 1 ? (STRPOOL_U32)config->block_capacity : 2U ); + pool->block_size = + (int) strpool_internal_pow2ceil( config->block_size > 256 ? (STRPOOL_U32)config->block_size : 256U ); + pool->min_data_size = + (int) ( sizeof( int ) * 2 + 1 + ( config->min_length > 8 ? (STRPOOL_U32)config->min_length : 8U ) ); + + pool->hash_capacity = pool->initial_entry_capacity * 2; + pool->entry_capacity = pool->initial_entry_capacity; + pool->handle_capacity = pool->initial_entry_capacity; + pool->block_capacity = pool->initial_block_capacity; + + pool->handle_freelist_head = -1; + pool->handle_freelist_tail = -1; + pool->block_count = 0; + pool->handle_count = 0; + pool->entry_count = 0; + + pool->hash_table = (strpool_internal_hash_slot_t*) STRPOOL_MALLOC( pool->memctx, + pool->hash_capacity * sizeof( *pool->hash_table ) ); + STRPOOL_ASSERT( pool->hash_table, "Allocation failed" ); + STRPOOL_MEMSET( pool->hash_table, 0, pool->hash_capacity * sizeof( *pool->hash_table ) ); + pool->entries = (strpool_internal_entry_t*) STRPOOL_MALLOC( pool->memctx, + pool->entry_capacity * sizeof( *pool->entries ) ); + STRPOOL_ASSERT( pool->entries, "Allocation failed" ); + pool->handles = (strpool_internal_handle_t*) STRPOOL_MALLOC( pool->memctx, + pool->handle_capacity * sizeof( *pool->handles ) ); + STRPOOL_ASSERT( pool->handles, "Allocation failed" ); + pool->blocks = (strpool_internal_block_t*) STRPOOL_MALLOC( pool->memctx, + pool->block_capacity * sizeof( *pool->blocks ) ); + STRPOOL_ASSERT( pool->blocks, "Allocation failed" ); + + pool->current_block = strpool_internal_add_block( pool, pool->block_size ); + } + + +void strpool_term( strpool_t* pool ) + { +#if 0 + // Debug statistics + printf( "\n\n" ); + printf( "Handles: %d/%d\n", pool->handle_count, pool->handle_capacity ); + printf( "Entries: %d/%d\n", pool->entry_count, pool->entry_capacity ); + printf( "Hashtable: %d/%d\n", pool->entry_count, pool->hash_capacity ); + printf( "Blocks: %d/%d\n", pool->block_count, pool->block_capacity ); + for( int i = 0; i < pool->block_count; ++i ) + { + printf( "\n" ); + printf( "BLOCK: %d\n", i ); + printf( "Capacity: %d\n", pool->blocks[ i ].capacity ); + printf( "Free: [ %d ]", (int)( pool->blocks[ i ].capacity - ( pool->blocks[ i ].tail - pool->blocks[ i ].data ) ) ); + int fl = pool->blocks[ i ].free_list; + int count = 0; + int size = 0; + int total = 0; + while( fl >= 0 ) + { + strpool_internal_free_block_t* free_entry = (strpool_internal_free_block_t*) ( pool->blocks[ i ].data + fl ); + total += free_entry->size; + if( size == 0 ) { size = free_entry->size; } + if( size != free_entry->size ) + { + printf( ", %dx%d", count, size ); + count = 1; + size = free_entry->size; + } + else + { + ++count; + } + fl = free_entry->next; + } + if( size != 0 ) printf( ", %dx%d", count, size ); + printf( ", { %d }\n", total ); + } + printf( "\n\n" ); +#endif + + for( int i = 0; i < pool->block_count; ++i ) STRPOOL_FREE( pool->memctx, pool->blocks[ i ].data ); + STRPOOL_FREE( pool->memctx, pool->blocks ); + STRPOOL_FREE( pool->memctx, pool->handles ); + STRPOOL_FREE( pool->memctx, pool->entries ); + STRPOOL_FREE( pool->memctx, pool->hash_table ); + } + + +void strpool_defrag( strpool_t* pool ) + { + int data_size = 0; + int count = 0; + for( int i = 0; i < pool->entry_count; ++i ) + { + strpool_internal_entry_t* entry = &pool->entries[ i ]; + if( entry->refcount > 0 ) + { + data_size += entry->size; + ++count; + } + } + + int data_capacity = data_size < pool->block_size ? + pool->block_size : (int)strpool_internal_pow2ceil( (STRPOOL_U32)data_size ); + + int hash_capacity = count + count / 2; + hash_capacity = hash_capacity < ( pool->initial_entry_capacity * 2 ) ? + ( pool->initial_entry_capacity * 2 ) : (int)strpool_internal_pow2ceil( (STRPOOL_U32)hash_capacity ); + strpool_internal_hash_slot_t* hash_table = (strpool_internal_hash_slot_t*) STRPOOL_MALLOC( pool->memctx, + hash_capacity * sizeof( *hash_table ) ); + STRPOOL_ASSERT( hash_table, "Allocation failed" ); + STRPOOL_MEMSET( hash_table, 0, hash_capacity * sizeof( *hash_table ) ); + + char* data = (char*) STRPOOL_MALLOC( pool->memctx, (size_t) data_capacity ); + STRPOOL_ASSERT( data, "Allocation failed" ); + int capacity = count < pool->initial_entry_capacity ? + pool->initial_entry_capacity : (int)strpool_internal_pow2ceil( (STRPOOL_U32)count ); + strpool_internal_entry_t* entries = (strpool_internal_entry_t*) STRPOOL_MALLOC( pool->memctx, + capacity * sizeof( *entries ) ); + STRPOOL_ASSERT( entries, "Allocation failed" ); + int index = 0; + char* tail = data; + for( int i = 0; i < pool->entry_count; ++i ) + { + strpool_internal_entry_t* entry = &pool->entries[ i ]; + if( entry->refcount > 0 ) + { + entries[ index ] = *entry; + + STRPOOL_U32 hash = pool->hash_table[ entry->hash_slot ].hash_key; + int base_slot = (int)( hash & (STRPOOL_U32)( hash_capacity - 1 ) ); + int slot = base_slot; + while( hash_table[ slot ].hash_key ) + slot = (slot + 1 ) & ( hash_capacity - 1 ); + STRPOOL_ASSERT( hash, "Invalid hash" ); + hash_table[ slot ].hash_key = hash; + hash_table[ slot ].entry_index = index; + ++hash_table[ base_slot ].base_count; + + entries[ index ].hash_slot = slot; + entries[ index ].data = tail; + entries[ index ].handle_index = entry->handle_index; + pool->handles[ entry->handle_index ].entry_index = index; + STRPOOL_MEMCPY( tail, entry->data, entry->length + 1 + 2 * sizeof( STRPOOL_U32 ) ); + tail += entry->size; + ++index; + } + } + + + STRPOOL_FREE( pool->memctx, pool->hash_table ); + STRPOOL_FREE( pool->memctx, pool->entries ); + for( int i = 0; i < pool->block_count; ++i ) STRPOOL_FREE( pool->memctx, pool->blocks[ i ].data ); + + if( pool->block_capacity != pool->initial_block_capacity ) + { + STRPOOL_FREE( pool->memctx, pool->blocks ); + pool->blocks = (strpool_internal_block_t*) STRPOOL_MALLOC( pool->memctx, + pool->initial_block_capacity * sizeof( *pool->blocks ) ); + STRPOOL_ASSERT( pool->blocks, "Allocation failed" ); + } + pool->block_capacity = pool->initial_block_capacity; + pool->block_count = 1; + pool->current_block = 0; + pool->blocks[ 0 ].capacity = data_capacity; + pool->blocks[ 0 ].data = data; + pool->blocks[ 0 ].tail = tail; + pool->blocks[ 0 ].free_list = -1; + + pool->hash_table = hash_table; + pool->hash_capacity = hash_capacity; + + pool->entries = entries; + pool->entry_capacity = capacity; + pool->entry_count = count; + } + + +static STRPOOL_U64 strpool_internal_make_handle( int index, int counter, STRPOOL_U64 index_mask, int counter_shift, + STRPOOL_U64 counter_mask ) + { + STRPOOL_U64 i = (STRPOOL_U64) ( index + 1 ); + STRPOOL_U64 c = (STRPOOL_U64) counter; + return ( ( c & counter_mask ) << counter_shift ) | ( i & index_mask ); + } + + +static int strpool_internal_counter_from_handle( STRPOOL_U64 handle, int counter_shift, STRPOOL_U64 counter_mask ) + { + return (int) ( ( handle >> counter_shift ) & counter_mask ) ; + } + + +static int strpool_internal_index_from_handle( STRPOOL_U64 handle, STRPOOL_U64 index_mask ) + { + return ( (int) ( handle & index_mask ) ) - 1; + } + + +static strpool_internal_entry_t* strpool_internal_get_entry( strpool_t const* pool, STRPOOL_U64 handle ) + { + int index = strpool_internal_index_from_handle( handle, pool->index_mask ); + int counter = strpool_internal_counter_from_handle( handle, pool->counter_shift, pool->counter_mask ); + + if( index >= 0 && index < pool->handle_count && + counter == (int) ( pool->handles[ index ].counter & pool->counter_mask ) ) + return &pool->entries[ pool->handles[ index ].entry_index ]; + + return 0; + } + + +static STRPOOL_U32 strpool_internal_find_in_blocks( strpool_t const* pool, char const* string, int length ) + { + for( int i = 0; i < pool->block_count; ++i ) + { + strpool_internal_block_t* block = &pool->blocks[ i ]; + // Check if string comes from pool + if( string >= block->data + 2 * sizeof( STRPOOL_U32 ) && string < block->data + block->capacity ) + { + STRPOOL_U32* ptr = (STRPOOL_U32*) string; + int stored_length = (int)( *( ptr - 1 ) ); // Length is stored immediately before string + if( stored_length != length || string[ length ] != '\0' ) return 0; // Invalid string + STRPOOL_U32 hash = *( ptr - 2 ); // Hash is stored before the length field + return hash; + } + } + + return 0; + } + + +static STRPOOL_U32 strpool_internal_calculate_hash( char const* string, int length, int ignore_case ) + { + STRPOOL_U32 hash = 5381U; + + if( ignore_case) + { + for( int i = 0; i < length; ++i ) + { + char c = string[ i ]; + c = ( c <= 'z' && c >= 'a' ) ? c - ( 'a' - 'A' ) : c; + hash = ( ( hash << 5U ) + hash) ^ c; + } + } + else + { + for( int i = 0; i < length; ++i ) + { + char c = string[ i ]; + hash = ( ( hash << 5U ) + hash) ^ c; + } + } + + hash = ( hash == 0 ) ? 1 : hash; // We can't allow 0-value hash keys, but dupes are ok + return hash; + } + + +static void strpool_internal_expand_hash_table( strpool_t* pool ) + { + int old_capacity = pool->hash_capacity; + strpool_internal_hash_slot_t* old_table = pool->hash_table; + + pool->hash_capacity *= 2; + + pool->hash_table = (strpool_internal_hash_slot_t*) STRPOOL_MALLOC( pool->memctx, + pool->hash_capacity * sizeof( *pool->hash_table ) ); + STRPOOL_ASSERT( pool->hash_table, "Allocation failed" ); + STRPOOL_MEMSET( pool->hash_table, 0, pool->hash_capacity * sizeof( *pool->hash_table ) ); + + for( int i = 0; i < old_capacity; ++i ) + { + STRPOOL_U32 hash_key = old_table[ i ].hash_key; + if( hash_key ) + { + int base_slot = (int)( hash_key & (STRPOOL_U32)( pool->hash_capacity - 1 ) ); + int slot = base_slot; + while( pool->hash_table[ slot ].hash_key ) + slot = ( slot + 1 ) & ( pool->hash_capacity - 1 ); + STRPOOL_ASSERT( hash_key, "Invalid hash" ); + pool->hash_table[ slot ].hash_key = hash_key; + pool->hash_table[ slot ].entry_index = old_table[ i ].entry_index; + pool->entries[ pool->hash_table[ slot ].entry_index ].hash_slot = slot; + ++pool->hash_table[ base_slot ].base_count; + } + } + + STRPOOL_FREE( pool->memctx, old_table ); + } + + +static void strpool_internal_expand_entries( strpool_t* pool ) + { + pool->entry_capacity *= 2; + strpool_internal_entry_t* new_entries = (strpool_internal_entry_t*) STRPOOL_MALLOC( pool->memctx, + pool->entry_capacity * sizeof( *pool->entries ) ); + STRPOOL_ASSERT( new_entries, "Allocation failed" ); + STRPOOL_MEMCPY( new_entries, pool->entries, pool->entry_count * sizeof( *pool->entries ) ); + STRPOOL_FREE( pool->memctx, pool->entries ); + pool->entries = new_entries; + } + + +static void strpool_internal_expand_handles( strpool_t* pool ) + { + pool->handle_capacity *= 2; + strpool_internal_handle_t* new_handles = (strpool_internal_handle_t*) STRPOOL_MALLOC( pool->memctx, + pool->handle_capacity * sizeof( *pool->handles ) ); + STRPOOL_ASSERT( new_handles, "Allocation failed" ); + STRPOOL_MEMCPY( new_handles, pool->handles, pool->handle_count * sizeof( *pool->handles ) ); + STRPOOL_FREE( pool->memctx, pool->handles ); + pool->handles = new_handles; + } + + +static char* strpool_internal_get_data_storage( strpool_t* pool, int size, int* alloc_size ) + { + if( (size_t)size < sizeof( strpool_internal_free_block_t ) ) size = sizeof( strpool_internal_free_block_t ); + if( size < pool->min_data_size ) size = pool->min_data_size; + size = (int)strpool_internal_pow2ceil( (STRPOOL_U32)size ); + + // Try to find a large enough free slot in existing blocks + for( int i = 0; i < pool->block_count; ++i ) + { + int free_list = pool->blocks[ i ].free_list; + int prev_list = -1; + while( free_list >= 0 ) + { + strpool_internal_free_block_t* free_entry = + (strpool_internal_free_block_t*) ( pool->blocks[ i ].data + free_list ); + if( free_entry->size / 2 < size ) + { + // At this point, all remaining slots are too small, so bail out if the current slot is not large enough + if( free_entry->size < size ) break; + + if( prev_list < 0 ) + { + pool->blocks[ i ].free_list = free_entry->next; + } + else + { + strpool_internal_free_block_t* prev_entry = + (strpool_internal_free_block_t*) ( pool->blocks[ i ].data + prev_list ); + prev_entry->next = free_entry->next; + } + *alloc_size = free_entry->size; + return (char*) free_entry; + } + prev_list = free_list; + free_list = free_entry->next; + } + } + + // Use current block, if enough space left + int offset = (int) ( pool->blocks[ pool->current_block ].tail - pool->blocks[ pool->current_block ].data ); + if( size <= pool->blocks[ pool->current_block ].capacity - offset ) + { + char* data = pool->blocks[ pool->current_block ].tail; + pool->blocks[ pool->current_block ].tail += size; + *alloc_size = size; + return data; + } + + // Allocate a new block + pool->current_block = strpool_internal_add_block( pool, size > pool->block_size ? size : pool->block_size ); + char* data = pool->blocks[ pool->current_block ].tail; + pool->blocks[ pool->current_block ].tail += size; + *alloc_size = size; + return data; + } + + +STRPOOL_U64 strpool_inject( strpool_t* pool, char const* string, int length ) + { + if( !string || length <= 0 ) return 0; + + STRPOOL_U32 hash = strpool_internal_find_in_blocks( pool, string, length ); + // If no stored hash, calculate it from data + if( !hash ) hash = strpool_internal_calculate_hash( string, length, pool->ignore_case ); + + // Return handle to existing string, if it is already in pool + int base_slot = (int)( hash & (STRPOOL_U32)( pool->hash_capacity - 1 ) ); + int base_count = pool->hash_table[ base_slot ].base_count; + int slot = base_slot; + int first_free = slot; + while( base_count > 0 ) + { + STRPOOL_U32 slot_hash = pool->hash_table[ slot ].hash_key; + if( slot_hash == 0 && pool->hash_table[ first_free ].hash_key != 0 ) first_free = slot; + int slot_base = (int)( slot_hash & (STRPOOL_U32)( pool->hash_capacity - 1 ) ); + if( slot_base == base_slot ) + { + STRPOOL_ASSERT( base_count > 0, "Invalid base count" ); + --base_count; + if( slot_hash == hash ) + { + int index = pool->hash_table[ slot ].entry_index; + strpool_internal_entry_t* entry = &pool->entries[ index ]; + if( entry->length == length && + ( + ( !pool->ignore_case && STRPOOL_MEMCMP( entry->data + 2 * sizeof( STRPOOL_U32 ), string, (size_t)length ) == 0 ) + || ( pool->ignore_case && STRPOOL_STRNICMP( entry->data + 2 * sizeof( STRPOOL_U32 ), string, (size_t)length ) == 0 ) + ) + ) + { + int handle_index = entry->handle_index; + return strpool_internal_make_handle( handle_index, pool->handles[ handle_index ].counter, + pool->index_mask, pool->counter_shift, pool->counter_mask ); + } + } + } + slot = ( slot + 1 ) & ( pool->hash_capacity - 1 ); + } + + // This is a new string, so let's add it + + if( pool->entry_count >= ( pool->hash_capacity - pool->hash_capacity / 3 ) ) + { + strpool_internal_expand_hash_table( pool ); + + base_slot = (int)( hash & (STRPOOL_U32)( pool->hash_capacity - 1 ) ); + slot = base_slot; + first_free = slot; + while( base_count ) + { + STRPOOL_U32 slot_hash = pool->hash_table[ slot ].hash_key; + if( slot_hash == 0 && pool->hash_table[ first_free ].hash_key != 0 ) first_free = slot; + int slot_base = (int)( slot_hash & (STRPOOL_U32)( pool->hash_capacity - 1 ) ); + if( slot_base == base_slot ) --base_count; + slot = ( slot + 1 ) & ( pool->hash_capacity - 1 ); + } + } + + slot = first_free; + while( pool->hash_table[ slot ].hash_key ) + slot = ( slot + 1 ) & ( pool->hash_capacity - 1 ); + + if( pool->entry_count >= pool->entry_capacity ) + strpool_internal_expand_entries( pool ); + + STRPOOL_ASSERT( !pool->hash_table[ slot ].hash_key && ( hash & ( (STRPOOL_U32) pool->hash_capacity - 1 ) ) == (STRPOOL_U32) base_slot, "Invalid slot" ); + STRPOOL_ASSERT( hash, "Invalid hash" ); + pool->hash_table[ slot ].hash_key = hash; + pool->hash_table[ slot ].entry_index = pool->entry_count; + ++pool->hash_table[ base_slot ].base_count; + + int handle_index; + + if( pool->handle_count < pool->handle_capacity ) + { + handle_index = pool->handle_count; + pool->handles[ pool->handle_count ].counter = 1; + ++pool->handle_count; + } + else if( pool->handle_freelist_head >= 0 ) + { + handle_index = pool->handle_freelist_head; + if( pool->handle_freelist_tail == pool->handle_freelist_head ) + pool->handle_freelist_tail = pool->handles[ pool->handle_freelist_head ].entry_index; + pool->handle_freelist_head = pool->handles[ pool->handle_freelist_head ].entry_index; + } + else + { + strpool_internal_expand_handles( pool ); + handle_index = pool->handle_count; + pool->handles[ pool->handle_count ].counter = 1; + ++pool->handle_count; + } + + pool->handles[ handle_index ].entry_index = pool->entry_count; + + strpool_internal_entry_t* entry = &pool->entries[ pool->entry_count ]; + ++pool->entry_count; + + int data_size = length + 1 + (int) ( 2 * sizeof( STRPOOL_U32 ) ); + char* data = strpool_internal_get_data_storage( pool, data_size, &data_size ); + entry->hash_slot = slot; + entry->handle_index = handle_index; + entry->data = data; + entry->size = data_size; + entry->length = length; + entry->refcount = 0; + + *(STRPOOL_U32*)(data) = hash; + data += sizeof( STRPOOL_U32 ); + *(STRPOOL_U32*)(data) = (STRPOOL_U32) length; + data += sizeof( STRPOOL_U32 ); + STRPOOL_MEMCPY( data, string, (size_t) length ); + data[ length ] = 0; // Ensure trailing zero + + return strpool_internal_make_handle( handle_index, pool->handles[ handle_index ].counter, pool->index_mask, + pool->counter_shift, pool->counter_mask ); + } + + +void strpool_discard( strpool_t* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t* entry = strpool_internal_get_entry( pool, handle ); + if( entry && entry->refcount == 0 ) + { + int entry_index = pool->handles[ entry->handle_index ].entry_index; + + // recycle string mem + for( int i = 0; i < pool->block_count; ++i ) + { + strpool_internal_block_t* block = &pool->blocks[ i ]; + if( entry->data >= block->data && entry->data <= block->tail ) + { + if( block->free_list < 0 ) + { + strpool_internal_free_block_t* new_entry = (strpool_internal_free_block_t*) ( entry->data ); + block->free_list = (int) ( entry->data - block->data ); + new_entry->next = -1; + new_entry->size = entry->size; + } + else + { + int free_list = block->free_list; + int prev_list = -1; + while( free_list >= 0 ) + { + strpool_internal_free_block_t* free_entry = + (strpool_internal_free_block_t*) ( pool->blocks[ i ].data + free_list ); + if( free_entry->size <= entry->size ) + { + strpool_internal_free_block_t* new_entry = (strpool_internal_free_block_t*) ( entry->data ); + if( prev_list < 0 ) + { + new_entry->next = pool->blocks[ i ].free_list; + pool->blocks[ i ].free_list = (int) ( entry->data - block->data ); + } + else + { + strpool_internal_free_block_t* prev_entry = + (strpool_internal_free_block_t*) ( pool->blocks[ i ].data + prev_list ); + prev_entry->next = (int) ( entry->data - block->data ); + new_entry->next = free_entry->next; + } + new_entry->size = entry->size; + break; + } + prev_list = free_list; + free_list = free_entry->next; + } + } + break; + } + } + + // recycle handle + if( pool->handle_freelist_tail < 0 ) + { + STRPOOL_ASSERT( pool->handle_freelist_head < 0, "Freelist error" ); + pool->handle_freelist_head = entry->handle_index; + pool->handle_freelist_tail = entry->handle_index; + } + else + { + pool->handles[ pool->handle_freelist_tail ].entry_index = entry->handle_index; + pool->handle_freelist_tail = entry->handle_index; + } + ++pool->handles[ entry->handle_index ].counter; // invalidate handle via counter + pool->handles[ entry->handle_index ].entry_index = -1; + + // recycle hash slot + STRPOOL_U32 hash = pool->hash_table[ entry->hash_slot ].hash_key; + int base_slot = (int)( hash & (STRPOOL_U32)( pool->hash_capacity - 1 ) ); + STRPOOL_ASSERT( hash, "Invalid hash" ); + --pool->hash_table[ base_slot ].base_count; + pool->hash_table[ entry->hash_slot ].hash_key = 0; + + // recycle entry + if( entry_index != pool->entry_count - 1 ) + { + pool->entries[ entry_index ] = pool->entries[ pool->entry_count - 1 ]; + pool->hash_table[ pool->entries[ entry_index ].hash_slot ].entry_index = entry_index; + pool->handles[ pool->entries[ entry_index ].handle_index ].entry_index = entry_index; + } + --pool->entry_count; + } + + } + + +int strpool_incref( strpool_t* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t* entry = strpool_internal_get_entry( pool, handle ); + if( entry ) + { + ++entry->refcount; + return entry->refcount; + } + return 0; + } + + +int strpool_decref( strpool_t* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t* entry = strpool_internal_get_entry( pool, handle ); + if( entry ) + { + STRPOOL_ASSERT( entry->refcount > 0, "Invalid ref count" ); + --entry->refcount; + return entry->refcount; + } + return 0; + } + + +int strpool_getref( strpool_t* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t* entry = strpool_internal_get_entry( pool, handle ); + if( entry ) return entry->refcount; + return 0; + } + + +int strpool_isvalid( strpool_t const* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t const* entry = strpool_internal_get_entry( pool, handle ); + if( entry ) return 1; + return 0; + } + + +char const* strpool_cstr( strpool_t const* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t const* entry = strpool_internal_get_entry( pool, handle ); + if( entry ) return entry->data + 2 * sizeof( STRPOOL_U32 ); // Skip leading hash value + return NULL; + } + + +int strpool_length( strpool_t const* pool, STRPOOL_U64 handle ) + { + strpool_internal_entry_t const* entry = strpool_internal_get_entry( pool, handle ); + if( entry ) return entry->length; + return 0; + } + + +char* strpool_collate( strpool_t const* pool, int* count ) + { + int size = 0; + for( int i = 0; i < pool->entry_count; ++i ) size += pool->entries[ i ].length + 1; + if( size == 0 ) return NULL; + + char* strings = (char*) STRPOOL_MALLOC( pool->memctx, (size_t) size ); + STRPOOL_ASSERT( strings, "Allocation failed" ); + *count = pool->entry_count; + char* ptr = strings; + for( int i = 0; i < pool->entry_count; ++i ) + { + int len = pool->entries[ i ].length + 1; + char* src = pool->entries[ i ].data + 2 * sizeof( STRPOOL_U32 ); + STRPOOL_MEMCPY( ptr, src, (size_t) len ); + ptr += len; + } + return strings; + } + + +void strpool_free_collated( strpool_t const* pool, char* collated_ptr ) + { + (void) pool; + STRPOOL_FREE( pool->memctx, collated_ptr ); + } + + +#endif /* STRPOOL_IMPLEMENTATION */ + + +/* +revision history: + 1.4 fixed find_in_blocks substring bug, removed realloc, added docs + 1.3 fixed typo in mask bit shift + 1.2 made it possible to override standard library functions + 1.1 added is_valid function to query a handles validity + 1.0 first released version +*/ + + +/* +------------------------------------------------------------------------------ + +This software is available under 2 licenses - you may choose the one you like. + +------------------------------------------------------------------------------ + +ALTERNATIVE A - MIT License + +Copyright (c) 2015 Mattias Gustavsson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------ + +ALTERNATIVE B - Public Domain (www.unlicense.org) + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------------------------------------------------------------------------------ +*/ diff --git a/ext/miniz/miniz.h b/ext/miniz/miniz.h index a0e7f692..26395855 100644 --- a/ext/miniz/miniz.h +++ b/ext/miniz/miniz.h @@ -115,9 +115,12 @@ */ #pragma once -// OpenJazz modification -#ifndef MINIZ_NO_ARCHIVE_APIS -#define MINIZ_NO_ARCHIVE_APIS +// OpenJazz Modification +#ifndef MINIZ_NO_ZLIB_COMPATIBLE_NAMES +# define MINIZ_NO_ZLIB_COMPATIBLE_NAMES +#endif +#if WORDS_BIGENDIAN==1 +# define MINIZ_LITTLE_ENDIAN 0 #endif #if defined(__3DS__) || defined(__wii__) # define OJ_NO_LARGE_FILE diff --git a/src/game/game.h b/src/game/game.h index c8985d5a..838c3fbc 100644 --- a/src/game/game.h +++ b/src/game/game.h @@ -159,7 +159,7 @@ class ServerGame : public Game { unsigned char recvBuffers[MAX_CLIENTS][BUFFER_LENGTH]; ///< Array of buffers containing data received from clients int received[MAX_CLIENTS]; ///< Array containing the amount of data received from each client unsigned char *levelData; ///< Contents of the current level file - int levelSize; ///< Size of the current level file + size_t levelSize; ///< Size of the current level file int sock; ///< Server socket public: diff --git a/src/io/file.cpp b/src/io/file.cpp index 3017ab24..80241692 100644 --- a/src/io/file.cpp +++ b/src/io/file.cpp @@ -33,11 +33,19 @@ #include #include + #if !(defined(_WIN32) || defined(WII) || defined(PSP) || defined(__3DS__)) #define UPPERCASE_FILENAMES #define LOWERCASE_FILENAMES #endif +#if WANT_ZIP +#include + +namespace { + static assetsys_t* asset_context = nullptr; +} +#endif /** * Try opening a file from the available paths. @@ -47,8 +55,44 @@ * @param write Whether or not the file can be written to */ File::File (const char* name, int pathType, bool write) : +#if WANT_ZIP + asset_loaded(false), asset_buffer(nullptr), asset_size(0), asset_pos(0), +#endif file(nullptr), filePath(nullptr) { +#if WANT_ZIP + if(!write && pathType & PATH_TYPE_GAME) { + assetsys_error_t err; + assetsys_file_t asset_file; + + // Create the file path for the given directory + filePath = createString("/GAME/", name); + + err = assetsys_file(asset_context, filePath, &asset_file); + if(err != ASSETSYS_SUCCESS) { + LOG_WARN("Could not find file: %s (%d)", name, err); + delete[] filePath; + throw E_FILE; + } + asset_size = assetsys_file_size(asset_context, asset_file); + asset_buffer = new char[asset_size]; + const char *f = assetsys_file_to_path(asset_context, asset_file); + + int loaded_size = 0; + err = assetsys_file_load(asset_context, asset_file, &loaded_size, asset_buffer, asset_size); + if(err != ASSETSYS_SUCCESS || (size_t)loaded_size != asset_size) { + delete[] asset_buffer; + asset_size = 0; + LOG_WARN("Could not open file: %s (%d)", f, err); + delete[] filePath; + throw E_FILE; + } + + LOG_TRACE("Opened file: %s", f); + asset_loaded = true; + return; + } +#endif forWriting = write; auto paths = gamePaths.getPaths(pathType); @@ -72,13 +116,18 @@ File::File (const char* name, int pathType, bool write) : * Delete the file object. */ File::~File () { - +#if WANT_ZIP + if (asset_loaded) { + delete[] asset_buffer; + asset_size = asset_pos = 0; + asset_loaded = false; + } else +#endif fclose(file); LOG_TRACE("Closed file: %s", filePath); delete[] filePath; - } @@ -121,7 +170,6 @@ bool File::open (std::string path, const char* name, bool write) { LOG_TRACE("Opened file: %s", filePath); return true; - } delete[] filePath; @@ -136,20 +184,18 @@ bool File::open (std::string path, const char* name, bool write) { * * @return The size of the file */ -int File::getSize () { - - int pos, size; - - pos = ftell(file); - +size_t File::getSize () { +#if WANT_ZIP + if (asset_loaded) { + return asset_size; + } +#endif + size_t pos = tell(); fseek(file, 0, SEEK_END); - - size = ftell(file); - - fseek(file, pos, SEEK_SET); + size_t size = tell(); + seek(pos, true); return size; - } @@ -158,10 +204,13 @@ int File::getSize () { * * @return The current location */ -int File::tell () { - +size_t File::tell () { +#if WANT_ZIP + if(asset_loaded) { + return asset_pos; + } +#endif return ftell(file); - } @@ -171,10 +220,14 @@ int File::tell () { * @param offset The new offset * @param reset Whether to offset from the current location or the start of the file */ -void File::seek (int offset, bool reset) { - +void File::seek (size_t offset, bool reset) { +#if WANT_ZIP + if(asset_loaded) { + asset_pos = reset ? offset : asset_pos + offset; + return; + } +#endif fseek(file, offset, reset ? SEEK_SET: SEEK_CUR); - } @@ -184,16 +237,19 @@ void File::seek (int offset, bool reset) { * @return The value read */ unsigned char File::loadChar () { - +#if WANT_ZIP + if(asset_loaded) { + if (asset_pos <= asset_size) + return asset_buffer[asset_pos++]; + return EOF; + } +#endif return fgetc(file); - } void File::storeChar (unsigned char val) { - fputc(val, file); - } @@ -203,14 +259,10 @@ void File::storeChar (unsigned char val) { * @return The value read */ unsigned short int File::loadShort () { - - unsigned short int val; - - val = fgetc(file); - val += fgetc(file) << 8; + unsigned short int val = loadChar(); + val += loadChar() << 8; return val; - } @@ -220,29 +272,19 @@ unsigned short int File::loadShort () { * @return The value read */ unsigned short int File::loadShort (unsigned short int max) { - - unsigned short int val; - - val = loadShort(); - + unsigned short int val = loadShort(); if (val > max) { - LOG_ERROR("Oversized value %d>%d in file %s", val, max, filePath); - return max; - } return val; - } void File::storeShort (unsigned short int val) { - fputc(val & 255, file); fputc(val >> 8, file); - } @@ -252,42 +294,32 @@ void File::storeShort (unsigned short int val) { * @return The value read */ signed int File::loadInt () { - - unsigned int val; - - val = fgetc(file); - val += fgetc(file) << 8; - val += fgetc(file) << 16; - val += fgetc(file) << 24; + unsigned int val = loadChar(); + val += loadChar() << 8; + val += loadChar() << 16; + val += loadChar() << 24; return *((signed int *)&val); - } void File::storeInt (signed int val) { - - unsigned int uval; - - uval = *((unsigned int *)&val); + unsigned int uval = *((unsigned int *)&val); fputc(uval & 255, file); fputc((uval >> 8) & 255, file); fputc((uval >> 16) & 255, file); fputc(uval >> 24, file); - } void File::storeData (void* data, int length) { - if(!forWriting) { LOG_ERROR("File %s not opened for writing!", filePath); return; } fwrite (data, length, 1, file); - } @@ -299,18 +331,26 @@ void File::storeData (void* data, int length) { * @return Buffer containing the block of data */ unsigned char * File::loadBlock (int length) { + unsigned char *buffer = new unsigned char[length]; - unsigned char *buffer; +#if WANT_ZIP + if(asset_loaded) { + int l = length; + if(asset_pos + length > asset_size) + l = asset_size - asset_pos; - buffer = new unsigned char[length]; + memcpy(buffer, (void*)(asset_buffer + asset_pos), l); + asset_pos += l; + return buffer; + } +#endif int res = fread(buffer, 1, length, file); if (res != length) LOG_ERROR("Could not read whole block (%d of %d bytes read)", res, length); return buffer; - } @@ -323,7 +363,8 @@ unsigned char * File::loadBlock (int length) { * @return Buffer containing the uncompressed data */ unsigned char* File::loadRLE (int length, bool checkSize) { - int start = 0, size = 0; + size_t start = 0; + int size = 0; if (checkSize) { // Determine the offset that follows the block @@ -348,21 +389,26 @@ unsigned char* File::loadRLE (int length, bool checkSize) { } else if (amount) { if (pos + amount >= length) break; +#if WANT_ZIP + for (int i = 0; i < amount; i++) { + buffer[pos++] = loadChar(); + } +#else fread(buffer + pos, 1, amount, file); pos += amount; +#endif } else buffer[pos++] = loadChar(); } if (checkSize) { if (tell() != start + size) - LOG_DEBUG("RLE block has incorrect size: %d vs. %d", tell() - start, size); + LOG_DEBUG("RLE block has incorrect size: %zu vs. %d", tell() - start, size); seek(start + size, true); } return buffer; - } @@ -370,14 +416,9 @@ unsigned char* File::loadRLE (int length, bool checkSize) { * Skip past a block of RLE compressed data in the file. */ void File::skipRLE () { + int next = loadShort(); - int next; - - next = fgetc(file); - next += fgetc(file) << 8; - - fseek(file, next, SEEK_CUR); - + seek(next, false); } @@ -413,14 +454,28 @@ unsigned char* File::loadLZ (int compressedLength, unsigned int length) { * @return The new string */ char * File::loadString () { - char *string; - int length = fgetc(file); + int length = loadChar(); if (length) { string = new char[length + 1]; + +#if WANT_ZIP + if(asset_loaded) { + int l = length; + if(asset_pos + length > asset_size) + l = asset_size - asset_pos; + + memcpy(string, (void*)(asset_buffer + asset_pos), l); + + asset_pos += l; + string[length] = 0; + return string; + } +#endif + int res = fread(string, 1, length, file); if (res != length) @@ -435,13 +490,13 @@ char * File::loadString () { int count; for (count = 0; count < 9; count++) { - string[count] = fgetc(file); + string[count] = loadChar(); if (string[count] == '.') { - string[++count] = fgetc(file); - string[++count] = fgetc(file); - string[++count] = fgetc(file); + string[++count] = loadChar(); + string[++count] = loadChar(); + string[++count] = loadChar(); count++; break; @@ -457,7 +512,6 @@ char * File::loadString () { string[length] = 0; return string; - } @@ -468,6 +522,21 @@ char * File::loadString () { */ char * File::loadString (int length) { char *string = new char[length + 1]; + +#if WANT_ZIP + if(asset_loaded) { + int l = length; + if(asset_pos + length > asset_size) + l = asset_size - asset_pos; + + memcpy(string, (void*)(asset_buffer + asset_pos), l); + + asset_pos += l; + string[length] = 0; + return string; + } +#endif + int res = fread(string, 1, length, file); if (res != length) @@ -560,7 +629,7 @@ unsigned char* File::loadPixels (int length, int key) { // Four pixels are packed into the lower end of each byte for (count = 0; count < length; count++) { - if (!(count & 3)) mask = fgetc(file); + if (!(count & 3)) mask = loadChar(); pixels[count] = (mask >> (count & 3)) & 1; } @@ -584,7 +653,7 @@ unsigned char* File::loadPixels (int length, int key) { // The unmasked portions are transparent, so no masked // portion should be transparent. - while (pixels[count] == key) pixels[count] = fgetc(file); + while (pixels[count] == key) pixels[count] = loadChar(); } @@ -632,6 +701,43 @@ void File::loadPalette (SDL_Color* palette, bool rle) { } +PathMgr::PathMgr() : + paths(), config(""), temp("") { +#if WANT_ZIP + asset_context = assetsys_create(nullptr); +#endif +} + +#if WANT_ZIP +static void list_assets( assetsys_t* assetsys, char const* path, int indent ) { + // Print folder names and recursively list assets + for( int i = 0; i < assetsys_subdir_count( assetsys, path ); ++i ) { + char const* subdir_name = assetsys_subdir_name( assetsys, path, i ); + for( int j = 0; j < indent; ++j ) printf( " " ); + printf( "%s/\n", subdir_name ); + + char const* subdir_path = assetsys_subdir_path( assetsys, path, i ); + list_assets( assetsys, subdir_path, indent + 1 ); + } + + // Print file names + for( int i = 0; i < assetsys_file_count( assetsys, path ); ++i ) { + char const* file_name = assetsys_file_name( assetsys, path, i ); + for( int j = 0; j < indent; ++j ) printf( " " ); + printf( "%s\n", file_name ); + } +} +#endif + +PathMgr::~PathMgr() { +#if WANT_ZIP + + //list_assets(asset_context, "/", 0); + + assetsys_destroy(asset_context); +#endif +} + bool PathMgr::add(char* newPath, int newPathType) { // Check for CWD @@ -648,11 +754,11 @@ bool PathMgr::add(char* newPath, int newPathType) { } // Append a directory separator if necessary - if (newPath[strlen(newPath) - 1] != OJ_DIR_SEP) { + /*if (newPath[strlen(newPath) - 1] != OJ_DIR_SEP) { char* tmp = createString(newPath, OJ_DIR_SEP_STR); delete[] newPath; newPath = tmp; - } + }*/ // all paths need to be readable if (access(newPath, R_OK) != 0) { @@ -705,6 +811,15 @@ bool PathMgr::add(char* newPath, int newPathType) { // Finally add paths.emplace_back(Path(newPath, newPathType)); +#if WANT_ZIP + //newPath[strlen(newPath) - 1] = '\0'; + if(newPathType & PATH_TYPE_GAME) { + assetsys_error_t err = assetsys_mount(asset_context, newPath, "/game"); + if(err != ASSETSYS_SUCCESS) { + LOG_WARN("Could not add path %s to asset loader.", newPath); + } + } +#endif delete[] newPath; return true; diff --git a/src/io/file.h b/src/io/file.h index c11de308..13eb813e 100644 --- a/src/io/file.h +++ b/src/io/file.h @@ -29,12 +29,23 @@ #include #include + +// ct +#define WANT_ZIP 1 + + // Classes /// File i/o class File { private: +#if WANT_ZIP + bool asset_loaded; + char* asset_buffer; + size_t asset_size; + size_t asset_pos; +#endif FILE* file; char* filePath; bool forWriting; @@ -45,9 +56,9 @@ class File { File (const char* name, int pathType, bool write = false); ~File (); - int getSize (); - void seek (int offset, bool reset = false); - int tell (); + size_t getSize (); + void seek (size_t offset, bool reset = false); + size_t tell (); unsigned char loadChar (); void storeChar (unsigned char val); unsigned short int loadShort (); @@ -100,7 +111,8 @@ class Path { class PathMgr { public: - PathMgr() = default; + PathMgr(); + ~PathMgr(); bool add(char* newPath, int newPathType); diff --git a/src/io/gfx/font.cpp b/src/io/gfx/font.cpp index 69367b32..e9bdd3e6 100644 --- a/src/io/gfx/font.cpp +++ b/src/io/gfx/font.cpp @@ -62,7 +62,7 @@ Font::Font (const char* fileName) { // Load font from a font file File* file = new File(fileName, PATH_TYPE_GAME); - int fileSize = file->getSize(); + size_t fileSize = file->getSize(); // Checking font file unsigned char *identifier1 = file->loadBlock(18); @@ -292,7 +292,7 @@ Font::Font (bool bonus) { // Load font from FONTS.000 or BONUS.000 File* file = new File(bonus? "BONUS.000": "FONTS.000", PATH_TYPE_GAME); - int fileSize = file->getSize(); + size_t fileSize = file->getSize(); nCharacters = file->loadShort(256); if (bonus) { diff --git a/src/io/sound.cpp b/src/io/sound.cpp index 43ff1091..bb13d093 100644 --- a/src/io/sound.cpp +++ b/src/io/sound.cpp @@ -276,7 +276,7 @@ void playMusic (const char * fileName, bool restart) { currentMusic = createString(fileName); // Find the size of the file - int size = file->getSize(); + size_t size = file->getSize(); // Read the entire file into memory file->seek(0, true); @@ -421,7 +421,7 @@ bool loadSounds (const char *fileName) { // Locate the header data file->seek(file->getSize() - 4, true); - int headerOffset = file->loadInt(); + size_t headerOffset = file->loadInt(); // Calculate number of sounds nRawSounds = (file->getSize() - headerOffset) / 18; diff --git a/src/jj1/bonuslevel/jj1bonuslevel.cpp b/src/jj1/bonuslevel/jj1bonuslevel.cpp index 46c825b7..68e9f5d8 100644 --- a/src/jj1/bonuslevel/jj1bonuslevel.cpp +++ b/src/jj1/bonuslevel/jj1bonuslevel.cpp @@ -79,7 +79,7 @@ int JJ1BonusLevel::loadSprites () { // Masked width <<= 2; - int pos = file->tell() + (pixelsLength << 2) + maskLength; + size_t pos = file->tell() + (pixelsLength << 2) + maskLength; // Read scrambled, masked pixel data pixels = file->loadPixels(width * height, 0); diff --git a/src/jj1/level/jj1levelload.cpp b/src/jj1/level/jj1levelload.cpp index 540e15cc..b34ad88d 100644 --- a/src/jj1/level/jj1levelload.cpp +++ b/src/jj1/level/jj1levelload.cpp @@ -114,7 +114,7 @@ int JJ1Level::loadPanel () { void JJ1Level::loadSprite (File* file, Sprite* sprite) { unsigned char* pixels; - int pos, maskOffset; + size_t pos, maskOffset; int width, height; // Load dimensions @@ -333,7 +333,7 @@ int JJ1Level::loadTiles (char* fileName) { file->seek(4, false); int pos = 0; - int fileSize = file->getSize(); + size_t fileSize = file->getSize(); // Read the RLE pixels // file::loadRLE() cannot be used, for reasons that will become clear diff --git a/src/jj1/scene/jj1scene.cpp b/src/jj1/scene/jj1scene.cpp index 9020ea4e..379422d4 100644 --- a/src/jj1/scene/jj1scene.cpp +++ b/src/jj1/scene/jj1scene.cpp @@ -249,7 +249,7 @@ JJ1Scene::JJ1Scene (const char * fileName) { dataOffsets.resize(dataItems); for (int i = 0; i < dataItems; i++) { dataOffsets[i] = file->loadInt(); // Load offset to script - LOG_MAX("dataOffsets: %ld", dataOffsets[i]); + LOG_MAX("dataOffsets: %zu", dataOffsets[i]); } loadData(file.get()); diff --git a/src/jj1/scene/jj1scene.h b/src/jj1/scene/jj1scene.h index d40b24a5..b42acfe8 100644 --- a/src/jj1/scene/jj1scene.h +++ b/src/jj1/scene/jj1scene.h @@ -210,10 +210,11 @@ class JJ1Scene { std::list fonts; /// Scripts all information needed to render script pages, text etc - std::vector pages; + std::vector pages; unsigned short int scriptItems, dataItems; - std::vector scriptStarts, dataOffsets; + std::vector scriptStarts; + std::vector dataOffsets; void loadScripts (File* f); void loadData (File* f); diff --git a/src/jj1/scene/jj1sceneload.cpp b/src/jj1/scene/jj1sceneload.cpp index 1ec24d99..8860384c 100644 --- a/src/jj1/scene/jj1sceneload.cpp +++ b/src/jj1/scene/jj1sceneload.cpp @@ -278,8 +278,8 @@ void JJ1Scene::loadAni (JJ1SceneAnimation &scene, File *f, int dataIndex) { } } else if (type == EPlayListAniHeader) { - int nextPos = f->tell(); - LOG_ANIM("PL Read position: %d", nextPos); + size_t nextPos = f->tell(); + LOG_ANIM("PL Read position: %zu", nextPos); f->loadShort(); // Length palettes.emplace_back(dataIndex); @@ -289,7 +289,7 @@ void JJ1Scene::loadAni (JJ1SceneAnimation &scene, File *f, int dataIndex) { #if DEBUG_ANIM int items = 0; #endif - LOG_ANIM("PL Read position start: %d", f->tell()); + LOG_ANIM("PL Read position start: %zu", f->tell()); int validValue = true; while (validValue) { @@ -406,7 +406,7 @@ void JJ1Scene::loadAni (JJ1SceneAnimation &scene, File *f, int dataIndex) { } - LOG_ANIM("PL Read position after block should be: %d", nextPos); + LOG_ANIM("PL Read position after block should be: %zu", nextPos); f->seek(nextPos, true); #if DEBUG_ANIM @@ -416,7 +416,7 @@ void JJ1Scene::loadAni (JJ1SceneAnimation &scene, File *f, int dataIndex) { } LOG_ANIM("PL Parsed through number of items skipping 0 items: %d", items); - LOG_ANIM("PL Read position after parsing anim blocks: %d", f->tell()); + LOG_ANIM("PL Read position after parsing anim blocks: %zu", f->tell()); } @@ -527,7 +527,7 @@ void JJ1Scene::loadScripts (File *f) { page.paletteIndex = palette; bool breakloop = false; - int pos = f->tell(); + size_t pos = f->tell(); int textPosX = -1; int textPosY = -1; @@ -812,7 +812,7 @@ void JJ1Scene::loadScripts (File *f) { case 0x3e: pos = f->tell(); - LOG_SCRIPT("Parse script end at position: %d, with: 0x%x", pos, type); + LOG_SCRIPT("Parse script end at position: %zu, with: 0x%x", pos, type); breakloop = true; f->loadChar(); @@ -821,7 +821,7 @@ void JJ1Scene::loadScripts (File *f) { default: pos = f->tell(); - LOG_SCRIPT("Parse script end at position: %d, breaker: 0x%x", pos, type); + LOG_SCRIPT("Parse script end at position: %zu, breaker: 0x%x", pos, type); breakloop = true; break; diff --git a/src/menu/mainmenu.cpp b/src/menu/mainmenu.cpp index 66d279fd..f10a6ba3 100644 --- a/src/menu/mainmenu.cpp +++ b/src/menu/mainmenu.cpp @@ -81,7 +81,7 @@ MainMenu::MainMenu () : } // only available in Holiday Hare 94/95 - if (file->getSize() > 200000) { + if (file->getSize() > 200000u) { time(¤tTime);