From 2d87349b8424211360b99c27f3d7f8bed0abc870 Mon Sep 17 00:00:00 2001 From: Ghabry Date: Mon, 12 Jun 2023 15:49:47 +0200 Subject: [PATCH 1/5] Support games in LZH archives --- CMakeLists.txt | 33 ++- Makefile.am | 4 + builds/cmake/Modules/Findlhasa.cmake | 64 +++++ configure.ac | 2 + src/filesystem.cpp | 12 +- src/filesystem_lzh.cpp | 392 +++++++++++++++++++++++++++ src/filesystem_lzh.h | 98 +++++++ src/filesystem_zip.cpp | 2 +- src/window_gamelist.cpp | 2 +- 9 files changed, 596 insertions(+), 13 deletions(-) create mode 100644 builds/cmake/Modules/Findlhasa.cmake create mode 100644 src/filesystem_lzh.cpp create mode 100644 src/filesystem_lzh.h diff --git a/CMakeLists.txt b/CMakeLists.txt index edb60d5de5..688eb2d970 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,8 @@ add_library(${PROJECT_NAME} OBJECT src/fileext_guesser.h src/filesystem.cpp src/filesystem.h + src/filesystem_lzh.cpp + src/filesystem_lzh.h src/filesystem_native.cpp src/filesystem_native.h src/filesystem_root.cpp @@ -827,7 +829,7 @@ endif() player_find_package(NAME PNG TARGET PNG::PNG REQUIRED) player_find_package(NAME fmt TARGET fmt::fmt REQUIRED) -# Do not use player_find_package. enable_language used by pixman on Android does work properly inside function calls +# Do not use player_find_package. enable_language used by pixman on Android does not work properly inside function calls find_package(Pixman REQUIRED) target_link_libraries(${PROJECT_NAME} PIXMAN::PIXMAN) @@ -857,6 +859,15 @@ if(TARGET freetype) CONFIG_BROKEN) endif() +# lzh archive support +option(PLAYER_WITH_LHASA "Support running games in lzh archives" ON) + +player_find_package(NAME lhasa + CONDITION PLAYER_WITH_LHASA + DEFINITION HAVE_LHASA + TARGET LHASA::liblhasa +) + # Sound system to use if(${PLAYER_TARGET_PLATFORM} STREQUAL "SDL2") set(PLAYER_AUDIO_BACKEND "SDL2" CACHE STRING "Audio system to use. Options: SDL2 OFF") @@ -1452,7 +1463,7 @@ if(${PLAYER_AUDIO_BACKEND} MATCHES "^(SDL2|SDL1|libretro|psvita|3ds|switch|wii)$ if(WAV_LIBS) message(STATUS "WAV playback: ${WAV_LIBS}") else() - message(STATUS "WAV playback: None") + message(STATUS "WAV playback: No") endif() set(MIDI_LIBS) @@ -1475,13 +1486,13 @@ if(${PLAYER_AUDIO_BACKEND} MATCHES "^(SDL2|SDL1|libretro|psvita|3ds|switch|wii)$ if(MIDI_LIBS) message(STATUS "MIDI playback: ${MIDI_LIBS}") else() - message(STATUS "MIDI playback: None") + message(STATUS "MIDI playback: No") endif() if(TARGET MPG123::libmpg123) message(STATUS "MP3 playback: mpg123") else() - message(STATUS "MP3 playback: None") + message(STATUS "MP3 playback: No") endif() if(TARGET Vorbis::vorbisfile) @@ -1489,19 +1500,19 @@ if(${PLAYER_AUDIO_BACKEND} MATCHES "^(SDL2|SDL1|libretro|psvita|3ds|switch|wii)$ elseif(TARGET Tremor::Tremor) message(STATUS "Ogg Vorbis playback: tremor") else() - message(STATUS "Ogg Vorbis playback: None") + message(STATUS "Ogg Vorbis playback: No") endif() if(TARGET XMP::XMP) message(STATUS "MOD playback: libxmp") else() - message(STATUS "MOD playback: None") + message(STATUS "MOD playback: No") endif() if(TARGET OpusFile::opusfile) message(STATUS "Opus playback: opusfile") else() - message(STATUS "Opus playback: None") + message(STATUS "Opus playback: No") endif() if(TARGET speexdsp::speexdsp) @@ -1509,7 +1520,7 @@ if(${PLAYER_AUDIO_BACKEND} MATCHES "^(SDL2|SDL1|libretro|psvita|3ds|switch|wii)$ elseif(TARGET Samplerate::Samplerate) message(STATUS "Resampler: libsamplerate") else() - message(STATUS "Resampler: None") + message(STATUS "Resampler: No") endif() endif() @@ -1523,6 +1534,12 @@ else() message(STATUS "Font rendering: built-in") endif() +if(TARGET LHASA::liblhasa) + message(STATUS "LZH archive support: lhasa") +else() + message(STATUS "LZH archive support: No") +endif() + message(STATUS "") message(STATUS "Manual page: ${MANUAL_STATUS}") diff --git a/Makefile.am b/Makefile.am index 2c2c339015..3c4246bd1b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,8 @@ libeasyrpg_player_a_SOURCES = \ src/fileext_guesser.h \ src/filesystem.cpp \ src/filesystem.h \ + src/filesystem_lzh.cpp \ + src/filesystem_lzh.h \ src/filesystem_native.cpp \ src/filesystem_native.h \ src/filesystem_root.cpp \ @@ -575,6 +577,7 @@ libeasyrpg_player_a_CXXFLAGS = \ $(PIXMAN_CFLAGS) \ $(FREETYPE_CFLAGS) \ $(HARFBUZZ_CFLAGS) \ + $(LHASA_CFLAGS) \ $(SDL_CFLAGS) \ $(PNG_CFLAGS) \ $(ZLIB_CFLAGS) \ @@ -627,6 +630,7 @@ easyrpg_player_LDADD = libeasyrpg-player.a libplayer-version.a \ $(PIXMAN_LIBS) \ $(FREETYPE_LIBS) \ $(HARFBUZZ_LIBS) \ + $(LHASA_LIBS) \ $(SDL_LIBS) \ $(PNG_LIBS) \ $(ZLIB_LIBS) \ diff --git a/builds/cmake/Modules/Findlhasa.cmake b/builds/cmake/Modules/Findlhasa.cmake new file mode 100644 index 0000000000..57d3a83a27 --- /dev/null +++ b/builds/cmake/Modules/Findlhasa.cmake @@ -0,0 +1,64 @@ +#.rst: +# Findlhasa +# ----------- +# +# Find the lhasa Library +# +# Imported Targets +# ^^^^^^^^^^^^^^^^ +# +# This module defines the following :prop_tgt:`IMPORTED` targets: +# +# ``LHASA::liblhasa`` +# The ``lhasa`` library, if found. +# +# Result Variables +# ^^^^^^^^^^^^^^^^ +# +# This module will set the following variables in your project: +# +# ``LHASA_INCLUDE_DIRS`` +# where to find lhasa headers. +# ``LHASA_LIBRARIES`` +# the libraries to link against to use lhasa. +# ``LHASA_FOUND`` +# true if the lhasa headers and libraries were found. + +find_package(PkgConfig QUIET) + +pkg_check_modules(PC_LHASA QUIET liblhasa) + +# Look for the header file. +find_path(LHASA_INCLUDE_DIR + NAMES lhasa.h + PATH_SUFFIXES liblhasa-1.0 liblhasa + HINTS ${PC_LHASA_INCLUDE_DIRS}) + +# Look for the library. +# Allow LHASA_LIBRARY to be set manually, as the location of the lhasa library +if(NOT LHASA_LIBRARY) + find_library(LHASA_LIBRARY + NAMES liblhasa lhasa + HINTS ${PC_LHASA_LIBRARY_DIRS}) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(lhasa + REQUIRED_VARS LHASA_LIBRARY LHASA_INCLUDE_DIR) + +if(LHASA_FOUND) + set(LHASA_INCLUDE_DIRS ${LHASA_INCLUDE_DIR}) + + if(NOT LHASA_LIBRARIES) + set(LHASA_LIBRARIES ${LHASA_LIBRARIES}) + endif() + + if(NOT TARGET LHASA::liblhasa) + add_library(LHASA::liblhasa UNKNOWN IMPORTED) + set_target_properties(LHASA::liblhasa PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LHASA_INCLUDE_DIRS}" + IMPORTED_LOCATION "${LHASA_LIBRARY}") + endif() +endif() + +mark_as_advanced(LHASA_INCLUDE_DIR LHASA_LIBRARY) diff --git a/configure.ac b/configure.ac index 64f9256f71..37e8fbbbc7 100644 --- a/configure.ac +++ b/configure.ac @@ -95,6 +95,7 @@ EP_PKG_CHECK([FREETYPE],[freetype2],[Custom Font rendering.]) AS_IF([test "$with_freetype" = "yes"],[ EP_PKG_CHECK([HARFBUZZ],[harfbuzz],[Custom Font text shaping.]) ]) +EP_PKG_CHECK([LHASA],[liblhasa],[Support running games in lzh archives.]) AC_ARG_WITH([audio],[AS_HELP_STRING([--without-audio], [Disable audio support. @<:@default=on@:>@])]) AS_IF([test "x$with_audio" != "xno"],[ @@ -226,6 +227,7 @@ if test "yes" != "$silent"; then echo " -custom Font rendering (freetype2): $with_freetype" test "$with_freetype" = "yes" && \ echo " -custom Font text shaping (harfbuzz): $with_harfbuzz" + echo " -run games in lzh archives (lhasa): $with_lhasa" if test "$with_audio" = "no"; then echo "Audio support: no" diff --git a/src/filesystem.cpp b/src/filesystem.cpp index 56a823317c..c06cec5999 100644 --- a/src/filesystem.cpp +++ b/src/filesystem.cpp @@ -17,6 +17,7 @@ #include "filesystem.h" #include "filesystem_native.h" +#include "filesystem_lzh.h" #include "filesystem_zip.h" #include "filesystem_stream.h" #include "filefinder.h" @@ -122,7 +123,7 @@ FilesystemView Filesystem::Create(StringView path) const { } else { path_prefix += comp + "/"; auto sv = StringView(comp); - if (sv.ends_with(".zip") || sv.ends_with(".easyrpg")) { + if (sv.ends_with(".zip") || sv.ends_with(".easyrpg") || sv.ends_with(".lzh")) { path_prefix.pop_back(); handle_internal = true; } @@ -133,9 +134,14 @@ FilesystemView Filesystem::Create(StringView path) const { internal_path.pop_back(); } - auto filesystem = std::make_shared(path_prefix, Subtree(dir_of_file)); + std::shared_ptr filesystem = std::make_shared(path_prefix, Subtree(dir_of_file)); if (!filesystem->IsValid()) { - return FilesystemView(); +#if HAVE_LHASA + filesystem = std::make_shared(path_prefix, Subtree(dir_of_file)); +#endif + if (!filesystem->IsValid()) { + return FilesystemView(); + } } if (!internal_path.empty()) { auto fs_view = filesystem->Create(internal_path); diff --git a/src/filesystem_lzh.cpp b/src/filesystem_lzh.cpp new file mode 100644 index 0000000000..c7339db7e8 --- /dev/null +++ b/src/filesystem_lzh.cpp @@ -0,0 +1,392 @@ +/* + * This file is part of EasyRPG Player. + * + * EasyRPG Player is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * EasyRPG Player is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with EasyRPG Player. If not, see . + */ + +#include "system.h" + +#ifdef HAVE_LHASA + +#include "filesystem_lzh.h" +#include "filefinder.h" +#include "output.h" +#include "utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lhasa.h" + +constexpr uint32_t end_of_central_directory = 0x06054b50; +constexpr int32_t end_of_central_directory_size = 22; + +constexpr uint32_t central_directory_entry = 0x02014b50; +constexpr uint32_t local_header = 0x04034b50; +constexpr uint32_t local_header_size = 30; + +static std::string normalize_path(StringView path) { + if (path == "." || path == "/" || path == "") { + return ""; + }; + std::string inner_path = FileFinder::MakeCanonical(path, 1); + std::replace(inner_path.begin(), inner_path.end(), '\\', '/'); + if (inner_path.front() == '.') { + inner_path = inner_path.substr(1, inner_path.size() - 1); + } + if (inner_path.front() == '/') { + inner_path = inner_path.substr(1, inner_path.size() - 1); + } + return inner_path; +} + +static int vio_read_func(void* handle, void* buf, size_t buf_len) { + auto* f = reinterpret_cast(handle); + if (buf_len == 0) return 0; + return f->read(reinterpret_cast(buf), buf_len).gcount(); +} + +static int vio_skip_func(void* handle, size_t bytes) { + auto* f = reinterpret_cast(handle); + f->seekg(bytes, std::ios_base::cur); + return 1; +} + +static size_t vio_read_dec_func(void* buf, size_t buf_len, void* user_data) { + auto* f = reinterpret_cast(user_data); + if (buf_len == 0) return 0; + f->read(reinterpret_cast(buf), buf_len); + return f->gcount(); +} + +static LHAInputStreamType vio = { + vio_read_func, + vio_skip_func, + nullptr // close not supported by istream interface +}; + +LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, StringView enc) : + Filesystem(base_path, parent_fs) { + is = parent_fs.OpenInputStream(GetPath()); + if (!is) { + return; + } + + lha_is.reset(lha_input_stream_new(&vio, &is)); + + lha_reader.reset(lha_reader_new(lha_is.get())); + + if (!lha_reader) { + return; + } + + encoding = ToString(enc); + LHAFileHeader* header; + + LzhEntry entry; + std::vector paths; + + // Compressed data offset is manually calculated to reduce calls to tellg() + size_t last_offset = is.tellg(); + + // TODO: Encoding detection + + while ((header = lha_reader_next_file(lha_reader.get())) != nullptr) { + std::string filepath; + + if (!strcmp(header->compress_method, LHA_COMPRESS_TYPE_DIR)) { + last_offset += header->raw_data_len; + + filepath = header->path; + if (filepath.back() == '/') { + filepath.pop_back(); + } + std::cout << "DIR: " << filepath << "\n"; + paths.push_back(filepath); + } else { + entry.uncompressed_size = header->length; + entry.compressed_size = header->compressed_length; + entry.fileoffset = last_offset + header->raw_data_len; + last_offset = entry.fileoffset + entry.compressed_size; + + std::cout << entry.fileoffset << " | " << is.tellg() << " | " << entry.uncompressed_size << "\n"; + + entry.is_directory = false; + entry.compress_method = header->compress_method; + if (header->path != nullptr) { + filepath = header->path; + if (filepath.back() == '/') { + paths.push_back(filepath); + paths.back().pop_back(); + } else { + paths.push_back(filepath); + filepath += '/'; + } + } + filepath += header->filename; + std::cout << "FILE: " << filepath << "\n"; + + lzh_entries.emplace_back(filepath, entry); + } + + // Determine intermediate directories + for (;;) { + filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); + if (filepath.empty()) { + break; + } + paths.push_back(filepath); + } + } + + /*if (encoding.empty()) { + zipfile.seekg(central_directory_offset); + std::stringstream filename_guess; + + // Guess the encoding first + int items = 0; + while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { + // Only consider Non-ASCII & Non-UTF8 for encoding detection + // Skip directories, files already contain the paths + if (is_utf8 || filepath.back() == '/' || Utils::StringIsAscii(filepath)) { + continue; + } + // Codepath will be only entered by Windows "compressed folder" ZIPs (uses local encoding) and + // 7zip (uses CP932 for Western European filenames) + + auto pos = filepath.find_last_of('/'); + if (pos == std::string::npos) { + filename_guess << filepath; + } else { + filename_guess << filepath.substr(pos + 1); + } + + ++items; + + if (items == 10) { + break; + } + } + + if (items == 0) { + // Only ASCII or UTF-8 flags set + encoding = "UTF-8"; + } else { + std::vector encodings = lcf::ReaderUtil::DetectEncodings(filename_guess.str()); + for (const auto &enc_ : encodings) { + std::string enc_test = lcf::ReaderUtil::Recode("\\", enc_); + if (enc_test.empty()) { + // Bad encoding + Output::Debug("Bad encoding: {}. Trying next.", enc_); + continue; + } + encoding = enc_; + break; + } + } + Output::Debug("Detected ZIP encoding: {}", encoding); + }*/ +/* + zipfile.clear(); + zipfile.seekg(central_directory_offset); + + std::vector paths; + while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { + if (is_utf8 || enc_is_utf8 || Utils::StringIsAscii(filepath)) { + // No reencoding necessary + filepath_cp437.clear(); + } else { + // also store CP437 to ensure files inside 7zip zip archives are found + filepath_cp437 = lcf::ReaderUtil::Recode(filepath, "437"); + filepath = lcf::ReaderUtil::Recode(filepath, encoding); + } + + // Workaround ZIP archives containing invalid "\" paths created by .net or Powershell + std::replace(filepath_cp437.begin(), filepath_cp437.end(), '\\', '/'); + std::replace(filepath.begin(), filepath.end(), '\\', '/'); + + // check if the entry is an directory or not (indicated by trailing /) + // this will fail when the (game) directory has cp437, but the users can rename it before + if (filepath.back() == '/') { + filepath = filepath.substr(0, filepath.size() - 1); + + // Determine intermediate directories + while (!filepath.empty()) { + paths.push_back(filepath); + filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); + } + } else { + lzh_entries.emplace_back(filepath, entry); + if (!filepath_cp437.empty()) { + lzh_entries_cp437.emplace_back(filepath_cp437, entry); + } + + // Determine intermediate directories + for (;;) { + filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); + if (filepath.empty()) { + break; + } + paths.push_back(filepath); + } + } + } + }*/ + + // Build directories + entry = {}; + entry.is_directory = true; + + // add root path + paths.emplace_back(""); + + std::sort(paths.begin(), paths.end()); + auto paths_del_it = std::unique(paths.begin(), paths.end()); + paths.erase(paths_del_it, paths.end()); + for (const auto& e : paths) { + lzh_entries.emplace_back(e, entry); + } + + // entries can be duplicated in the lzh archive, e.g. when creating a game disk the RTP is embedded, followed by + // the game entries. Use a stable sort to preserve this order. + std::stable_sort(lzh_entries.begin(), lzh_entries.end(), [](auto& a, auto& b) { + return a.first < b.first; + }); + + // Then remove all duplicates but keep the last + auto entries_del_it = std::unique(lzh_entries.rbegin(), lzh_entries.rend(), [](auto& a, auto& b) { + return a.first < b.first; + }); + lzh_entries.erase(lzh_entries.begin(), entries_del_it.base()); +} + +bool LzhFilesystem::IsFile(StringView path) const { + std::string path_normalized = normalize_path(path); + auto entry = Find(path); + if (entry) { + return !entry->is_directory; + } + return false; +} + +bool LzhFilesystem::IsDirectory(StringView path, bool) const { + std::string path_normalized = normalize_path(path); + auto entry = Find(path); + if (entry) { + return entry->is_directory; + } + return false; +} + +bool LzhFilesystem::Exists(StringView path) const { + std::string path_normalized = normalize_path(path); + auto entry = Find(path); + return entry != nullptr; +} + +int64_t LzhFilesystem::GetFilesize(StringView path) const { + std::string path_normalized = normalize_path(path); + auto entry = Find(path); + if (entry) { + return entry->uncompressed_size; + } + return 0; +} + +std::streambuf* LzhFilesystem::CreateInputStreambuffer(StringView path, std::ios_base::openmode) const { + std::string path_normalized = normalize_path(path); + auto entry = Find(path); + if (entry && !entry->is_directory) { + // Determine compression method + auto* decoder_type = lha_decoder_for_name(const_cast(entry->compress_method.c_str())); + + if (!decoder_type) { + // TODO Error + return nullptr; + } + + // Seek to the compressed data + is.clear(); + is.seekg(entry->fileoffset, std::ios_base::beg); + + // Create a suitable decoder for the compression method + std::unique_ptr decoder; + decoder.reset(lha_decoder_new(decoder_type, vio_read_dec_func, &is, entry->uncompressed_size)); + + // Decompress + auto dec_buf = std::vector(entry->uncompressed_size); + size_t res = lha_decoder_read(decoder.get(), dec_buf.data(), dec_buf.size()); + + // TODO: Error handling + + return new Filesystem_Stream::InputMemoryStreamBuf(std::move(dec_buf)); + } + + return nullptr; +} + +bool LzhFilesystem::GetDirectoryContent(StringView path, std::vector& entries) const { + if (!IsDirectory(path, false)) { + return false; + } + + std::string path_normalized = normalize_path(path); + if (!path_normalized.empty() && path_normalized.back() != '/') { + path_normalized += "/"; + } + + auto check = [&](auto& it) { + if (StringView(it.first).starts_with(path_normalized) && + it.first.substr(path_normalized.size(), it.first.size() - path_normalized.size()).find_last_of('/') == std::string::npos) { + // Everything that starts with the path but isn't the path and does contain no slash + auto filename = it.first.substr(path_normalized.size(), it.first.size() - path_normalized.size()); + if (filename.empty()) { + return; + } + + entries.emplace_back( + it.first.substr(path_normalized.size(), it.first.size() - path_normalized.size()), + it.second.is_directory ? DirectoryTree::FileType::Directory : DirectoryTree::FileType::Regular); + } + }; + + for (const auto& it : lzh_entries) { + check(it); + } + + return true; +} + +const LzhFilesystem::LzhEntry* LzhFilesystem::Find(StringView what) const { + auto it = std::lower_bound(lzh_entries.begin(), lzh_entries.end(), what, [](const auto& e, const auto& w) { + return e.first < w; + }); + if (it != lzh_entries.end() && it->first == what) { + return &it->second; + } + + return nullptr; +} + +std::string LzhFilesystem::Describe() const { + return fmt::format("[LZH] {} ({})", GetPath(), encoding); +} + +#endif diff --git a/src/filesystem_lzh.h b/src/filesystem_lzh.h new file mode 100644 index 0000000000..0f5eaf352a --- /dev/null +++ b/src/filesystem_lzh.h @@ -0,0 +1,98 @@ +/* + * This file is part of EasyRPG Player. + * + * EasyRPG Player is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * EasyRPG Player is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with EasyRPG Player. If not, see . + */ + +#ifndef EP_FILESYSTEM_LZH_H +#define EP_FILESYSTEM_LZH_H + +#include "system.h" + +#ifdef HAVE_LHASA + +#include "filesystem.h" +#include "filesystem_stream.h" +#include +#include +#include +#include + +#include + +/** + * A virtual filesystem that allows file/directory operations inside a LZH archive. + */ +class LzhFilesystem : public Filesystem { +public: + /** + * Initializes a filesystem inside the given LZH File + * + * @param base_path Path passed to parent_fs to open the LZH file + * @param parent_fs Filesystem used to create handles on the LZH file + * @param encoding Encoding to use, use empty string for autodetection + */ + LzhFilesystem(std::string base_path, FilesystemView parent_fs, StringView encoding = ""); + +protected: + /** + * Implementation of abstract methods + */ + /** @{ */ + bool IsFile(StringView path) const override; + bool IsDirectory(StringView path, bool follow_symlinks) const override; + bool Exists(StringView path) const override; + int64_t GetFilesize(StringView path) const override; + std::streambuf* CreateInputStreambuffer(StringView path, std::ios_base::openmode mode) const override; + bool GetDirectoryContent(StringView path, std::vector& entries) const override; + std::string Describe() const override; + /** @} */ + +private: + struct LzhEntry { + size_t compressed_size; + size_t uncompressed_size; + size_t fileoffset; + std::string compress_method; + bool is_directory; + }; + + const LzhEntry* Find(StringView what) const; + + std::vector> lzh_entries; + std::string encoding; + mutable std::vector filename_buffer; + + struct LhasaDeleter { + void operator()(LHAInputStream* o) const { + lha_input_stream_free(o); + } + + void operator()(LHAReader* o) const { + lha_reader_free(o); + } + + void operator()(LHADecoder* o) const { + lha_decoder_free(o); + } + }; + + mutable Filesystem_Stream::InputStream is; + mutable std::unique_ptr lha_is; + mutable std::unique_ptr lha_reader; +}; + +#endif + +#endif diff --git a/src/filesystem_zip.cpp b/src/filesystem_zip.cpp index 7c0e4cf674..68918526d9 100644 --- a/src/filesystem_zip.cpp +++ b/src/filesystem_zip.cpp @@ -184,7 +184,7 @@ ZipFilesystem::ZipFilesystem(std::string base_path, FilesystemView parent_fs, St return a.first < b.first; }); } else { - Output::Warning("ZipFS: {} is not a valid archive", GetPath()); + Output::Debug("ZipFS: {} is not a valid archive", GetPath()); } } diff --git a/src/window_gamelist.cpp b/src/window_gamelist.cpp index eec944888b..8f0ad35878 100644 --- a/src/window_gamelist.cpp +++ b/src/window_gamelist.cpp @@ -55,7 +55,7 @@ bool Window_GameList::Refresh(FilesystemView filesystem_base, bool show_dotdot) } if (dir.second.type == DirectoryTree::FileType::Regular) { auto sv = StringView(dir.second.name); - if (sv.ends_with(".zip") || sv.ends_with(".easyrpg")) { + if (sv.ends_with(".zip") || sv.ends_with(".easyrpg") || sv.ends_with(".lzh")) { game_directories.emplace_back(dir.second.name); } } else if (dir.second.type == DirectoryTree::FileType::Directory) { From 6576df186e3982b10c3d3e1382d46a5bb7d11698 Mon Sep 17 00:00:00 2001 From: Ghabry Date: Wed, 25 Oct 2023 23:25:16 +0200 Subject: [PATCH 2/5] String Var: Swap Decode and Encode Encode converts to UTF-8 and Decode from UTF-8. These functions in liblcf need a better name or a documentation comment. --- src/game_strings.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/game_strings.cpp b/src/game_strings.cpp index 913a596a65..6a37efb253 100644 --- a/src/game_strings.cpp +++ b/src/game_strings.cpp @@ -127,7 +127,7 @@ Game_Strings::Str_t Game_Strings::FromFile(StringView filename, int encoding, bo if (encoding == 0) { lcf::Encoder enc(Player::encoding); - enc.Decode(file_content); + enc.Encode(file_content); } return file_content; @@ -160,7 +160,7 @@ Game_Strings::Str_t Game_Strings::ToFile(Str_Params params, std::string filename if (encoding == 0) { lcf::Encoder enc(Player::encoding); - enc.Encode(str); + enc.Decode(str); } txt_out << str; From e5ccd12ff246353c84586d808370ecaba30d1464 Mon Sep 17 00:00:00 2001 From: Ghabry Date: Wed, 25 Oct 2023 23:27:16 +0200 Subject: [PATCH 3/5] LzhFS: Finalize the code and add encoding detection --- src/filefinder.cpp | 15 +++- src/filefinder.h | 8 ++ src/filesystem.cpp | 11 ++- src/filesystem_lzh.cpp | 191 ++++++++++++++++------------------------ src/filesystem_lzh.h | 2 +- src/window_gamelist.cpp | 3 +- 6 files changed, 104 insertions(+), 126 deletions(-) diff --git a/src/filefinder.cpp b/src/filefinder.cpp index b7885e48a5..475f2560eb 100644 --- a/src/filefinder.cpp +++ b/src/filefinder.cpp @@ -90,7 +90,7 @@ FilesystemView FileFinder::Save() { if (!game_fs) { // Filesystem not initialized yet (happens on startup) - return FilesystemView(); + return {}; } // Not overwritten, check if game fs is writable. If not redirect the write operation. @@ -285,6 +285,19 @@ std::string FileFinder::GetPathInsideGamePath(StringView path_in) { return FileFinder::GetPathInsidePath(Game().GetFullPath(), path_in); } +bool FileFinder::IsSupportedArchiveExtension(std::string path) { + Utils::LowerCaseInPlace(path); + StringView pv = path; + +#ifdef HAVE_LHASA + if (pv.ends_with(".lzh")) { + return true; + } +#endif + + return pv.ends_with(".zip") || pv.ends_with(".easyrpg"); +} + void FileFinder::Quit() { root_fs.reset(); } diff --git a/src/filefinder.h b/src/filefinder.h index 8f6be1ed68..125a305bb6 100644 --- a/src/filefinder.h +++ b/src/filefinder.h @@ -244,6 +244,14 @@ namespace FileFinder { */ std::string GetPathInsideGamePath(StringView path_in); + /** + * Checks whether a passed path ends with a supported extension for an archive, e.g. ".zip" + * + * @param path path to check + * @return true when the path ends on an archive extension + */ + bool IsSupportedArchiveExtension(std::string path); + /** * @param p tree Tree to check * @return Whether the tree contains a valid RPG2k(3) or EasyRPG project diff --git a/src/filesystem.cpp b/src/filesystem.cpp index c06cec5999..132558f60c 100644 --- a/src/filesystem.cpp +++ b/src/filesystem.cpp @@ -122,8 +122,7 @@ FilesystemView Filesystem::Create(StringView path) const { internal_path += comp + "/"; } else { path_prefix += comp + "/"; - auto sv = StringView(comp); - if (sv.ends_with(".zip") || sv.ends_with(".easyrpg") || sv.ends_with(".lzh")) { + if (FileFinder::IsSupportedArchiveExtension(comp)) { path_prefix.pop_back(); handle_internal = true; } @@ -140,13 +139,13 @@ FilesystemView Filesystem::Create(StringView path) const { filesystem = std::make_shared(path_prefix, Subtree(dir_of_file)); #endif if (!filesystem->IsValid()) { - return FilesystemView(); + return {}; } } if (!internal_path.empty()) { auto fs_view = filesystem->Create(internal_path); if (!fs_view) { - return FilesystemView(); + return {}; } return fs_view; } @@ -155,7 +154,7 @@ FilesystemView Filesystem::Create(StringView path) const { // This way archives with structure "archive/game_folder" launch the game directly auto fs_view = filesystem->Subtree(""); if (!fs_view) { - return FilesystemView(); + return {}; } auto entries = fs_view.ListDirectory(""); if (entries->size() == 1 && entries->begin()->second.type == DirectoryTree::FileType::Directory) { @@ -164,7 +163,7 @@ FilesystemView Filesystem::Create(StringView path) const { return fs_view; } else { if (!(Exists(path) || !IsDirectory(path, true))) { - return FilesystemView(); + return {}; } // Handle as a normal path in the current filesystem diff --git a/src/filesystem_lzh.cpp b/src/filesystem_lzh.cpp index c7339db7e8..9489e6994e 100644 --- a/src/filesystem_lzh.cpp +++ b/src/filesystem_lzh.cpp @@ -24,26 +24,18 @@ #include "output.h" #include "utils.h" -#include +#include #include #include #include #include -#include #include #include #include "lhasa.h" -constexpr uint32_t end_of_central_directory = 0x06054b50; -constexpr int32_t end_of_central_directory_size = 22; - -constexpr uint32_t central_directory_entry = 0x02014b50; -constexpr uint32_t local_header = 0x04034b50; -constexpr uint32_t local_header_size = 30; - static std::string normalize_path(StringView path) { - if (path == "." || path == "/" || path == "") { + if (path == "." || path == "/" || path.empty()) { return ""; }; std::string inner_path = FileFinder::MakeCanonical(path, 1); @@ -94,6 +86,7 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St lha_reader.reset(lha_reader_new(lha_is.get())); if (!lha_reader) { + Output::Debug("LzhFS: {} is not a valid archive", GetPath()); return; } @@ -104,9 +97,63 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St std::vector paths; // Compressed data offset is manually calculated to reduce calls to tellg() - size_t last_offset = is.tellg(); + auto last_offset = is.tellg(); + + // Guess the encoding + if (encoding.empty()) { + std::stringstream filename_guess; + int items = 0; + + while ((header = lha_reader_next_file(lha_reader.get())) != nullptr) { + std::string filepath; + + // Only consider Non-ASCII and skip directories + if (strcmp(header->compress_method, LHA_COMPRESS_TYPE_DIR) != 0) { + filepath = header->filename; + if (Utils::StringIsAscii(filepath)) { + continue; + } + filename_guess << filepath; + + ++items; + if (items == 10) { + break; + } + } + } + + if (items == 0) { + // Only ASCII text (UTF-8 compatible) + encoding = "UTF-8"; + } else { + std::vector encodings = lcf::ReaderUtil::DetectEncodings(filename_guess.str()); + for (const auto &enc_ : encodings) { + lcf::Encoder lcf_encoder(enc_); + if (!lcf_encoder.IsOk()) { + // Bad encoding + Output::Debug("Bad encoding: {}. Trying next.", enc_); + continue; + } + encoding = enc_; + break; + } + } + Output::Debug("Detected LZH encoding: {}", encoding); + + is.clear(); + is.seekg(last_offset); + + // Cannot figure out how to rewind the reader: Creating a new one instead + lha_reader.reset(lha_reader_new(lha_is.get())); - // TODO: Encoding detection + if (!lha_reader) { + Output::Debug("LzhFS: {} is not a valid archive", GetPath()); + return; + } + } + + // Read the archive + lcf::Encoder lzh_encoder(encoding); while ((header = lha_reader_next_file(lha_reader.get())) != nullptr) { std::string filepath; @@ -115,23 +162,25 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St last_offset += header->raw_data_len; filepath = header->path; + lzh_encoder.Encode(filepath); if (filepath.back() == '/') { filepath.pop_back(); } - std::cout << "DIR: " << filepath << "\n"; paths.push_back(filepath); } else { entry.uncompressed_size = header->length; entry.compressed_size = header->compressed_length; - entry.fileoffset = last_offset + header->raw_data_len; + entry.fileoffset = last_offset + static_cast(header->raw_data_len); last_offset = entry.fileoffset + entry.compressed_size; - std::cout << entry.fileoffset << " | " << is.tellg() << " | " << entry.uncompressed_size << "\n"; - entry.is_directory = false; entry.compress_method = header->compress_method; if (header->path != nullptr) { + // File is not in the root filepath = header->path; + lzh_encoder.Encode(filepath); + + // Safety check: Directories should end with a / if (filepath.back() == '/') { paths.push_back(filepath); paths.back().pop_back(); @@ -140,8 +189,9 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St filepath += '/'; } } - filepath += header->filename; - std::cout << "FILE: " << filepath << "\n"; + std::string fname = header->filename; + lzh_encoder.Encode(fname); + filepath += fname; lzh_entries.emplace_back(filepath, entry); } @@ -156,100 +206,6 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St } } - /*if (encoding.empty()) { - zipfile.seekg(central_directory_offset); - std::stringstream filename_guess; - - // Guess the encoding first - int items = 0; - while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { - // Only consider Non-ASCII & Non-UTF8 for encoding detection - // Skip directories, files already contain the paths - if (is_utf8 || filepath.back() == '/' || Utils::StringIsAscii(filepath)) { - continue; - } - // Codepath will be only entered by Windows "compressed folder" ZIPs (uses local encoding) and - // 7zip (uses CP932 for Western European filenames) - - auto pos = filepath.find_last_of('/'); - if (pos == std::string::npos) { - filename_guess << filepath; - } else { - filename_guess << filepath.substr(pos + 1); - } - - ++items; - - if (items == 10) { - break; - } - } - - if (items == 0) { - // Only ASCII or UTF-8 flags set - encoding = "UTF-8"; - } else { - std::vector encodings = lcf::ReaderUtil::DetectEncodings(filename_guess.str()); - for (const auto &enc_ : encodings) { - std::string enc_test = lcf::ReaderUtil::Recode("\\", enc_); - if (enc_test.empty()) { - // Bad encoding - Output::Debug("Bad encoding: {}. Trying next.", enc_); - continue; - } - encoding = enc_; - break; - } - } - Output::Debug("Detected ZIP encoding: {}", encoding); - }*/ -/* - zipfile.clear(); - zipfile.seekg(central_directory_offset); - - std::vector paths; - while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { - if (is_utf8 || enc_is_utf8 || Utils::StringIsAscii(filepath)) { - // No reencoding necessary - filepath_cp437.clear(); - } else { - // also store CP437 to ensure files inside 7zip zip archives are found - filepath_cp437 = lcf::ReaderUtil::Recode(filepath, "437"); - filepath = lcf::ReaderUtil::Recode(filepath, encoding); - } - - // Workaround ZIP archives containing invalid "\" paths created by .net or Powershell - std::replace(filepath_cp437.begin(), filepath_cp437.end(), '\\', '/'); - std::replace(filepath.begin(), filepath.end(), '\\', '/'); - - // check if the entry is an directory or not (indicated by trailing /) - // this will fail when the (game) directory has cp437, but the users can rename it before - if (filepath.back() == '/') { - filepath = filepath.substr(0, filepath.size() - 1); - - // Determine intermediate directories - while (!filepath.empty()) { - paths.push_back(filepath); - filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); - } - } else { - lzh_entries.emplace_back(filepath, entry); - if (!filepath_cp437.empty()) { - lzh_entries_cp437.emplace_back(filepath_cp437, entry); - } - - // Determine intermediate directories - for (;;) { - filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); - if (filepath.empty()) { - break; - } - paths.push_back(filepath); - } - } - } - }*/ - // Build directories entry = {}; entry.is_directory = true; @@ -264,7 +220,7 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St lzh_entries.emplace_back(e, entry); } - // entries can be duplicated in the lzh archive, e.g. when creating a game disk the RTP is embedded, followed by + // entries can be duplicated in the archive, e.g. when creating a game disk the RTP is embedded, followed by // the game entries. Use a stable sort to preserve this order. std::stable_sort(lzh_entries.begin(), lzh_entries.end(), [](auto& a, auto& b) { return a.first < b.first; @@ -272,7 +228,7 @@ LzhFilesystem::LzhFilesystem(std::string base_path, FilesystemView parent_fs, St // Then remove all duplicates but keep the last auto entries_del_it = std::unique(lzh_entries.rbegin(), lzh_entries.rend(), [](auto& a, auto& b) { - return a.first < b.first; + return a.first == b.first; }); lzh_entries.erase(lzh_entries.begin(), entries_del_it.base()); } @@ -318,7 +274,7 @@ std::streambuf* LzhFilesystem::CreateInputStreambuffer(StringView path, std::ios auto* decoder_type = lha_decoder_for_name(const_cast(entry->compress_method.c_str())); if (!decoder_type) { - // TODO Error + Output::Warning("LzhFS: Unsupported compression method {} for {}", entry->compress_method, path_normalized); return nullptr; } @@ -334,7 +290,10 @@ std::streambuf* LzhFilesystem::CreateInputStreambuffer(StringView path, std::ios auto dec_buf = std::vector(entry->uncompressed_size); size_t res = lha_decoder_read(decoder.get(), dec_buf.data(), dec_buf.size()); - // TODO: Error handling + if (res != entry->uncompressed_size) { + Output::Warning("LzhFS: Less data compressed than expected ({})", path_normalized); + return nullptr; + } return new Filesystem_Stream::InputMemoryStreamBuf(std::move(dec_buf)); } diff --git a/src/filesystem_lzh.h b/src/filesystem_lzh.h index 0f5eaf352a..6bfc17dff2 100644 --- a/src/filesystem_lzh.h +++ b/src/filesystem_lzh.h @@ -63,7 +63,7 @@ class LzhFilesystem : public Filesystem { struct LzhEntry { size_t compressed_size; size_t uncompressed_size; - size_t fileoffset; + std::streamoff fileoffset; std::string compress_method; bool is_directory; }; diff --git a/src/window_gamelist.cpp b/src/window_gamelist.cpp index 8f0ad35878..bf6b58fed0 100644 --- a/src/window_gamelist.cpp +++ b/src/window_gamelist.cpp @@ -54,8 +54,7 @@ bool Window_GameList::Refresh(FilesystemView filesystem_base, bool show_dotdot) continue; } if (dir.second.type == DirectoryTree::FileType::Regular) { - auto sv = StringView(dir.second.name); - if (sv.ends_with(".zip") || sv.ends_with(".easyrpg") || sv.ends_with(".lzh")) { + if (FileFinder::IsSupportedArchiveExtension(dir.second.name)) { game_directories.emplace_back(dir.second.name); } } else if (dir.second.type == DirectoryTree::FileType::Directory) { From 87d03e9d38c1e0ef1145a80bd2306db0c8fe7191 Mon Sep 17 00:00:00 2001 From: Ghabry Date: Wed, 25 Oct 2023 23:28:18 +0200 Subject: [PATCH 4/5] ZipFS: Port some enhancements from LzhFS over Fix #3094 --- src/filesystem_zip.cpp | 221 ++++++++++++++++++++++------------------- 1 file changed, 120 insertions(+), 101 deletions(-) diff --git a/src/filesystem_zip.cpp b/src/filesystem_zip.cpp index 68918526d9..b436a71e9a 100644 --- a/src/filesystem_zip.cpp +++ b/src/filesystem_zip.cpp @@ -21,6 +21,7 @@ #include "utils.h" #include +#include #include #include #include @@ -37,7 +38,7 @@ constexpr uint32_t local_header = 0x04034b50; constexpr uint32_t local_header_size = 30; static std::string normalize_path(StringView path) { - if (path == "." || path == "/" || path == "") { + if (path == "." || path == "/" || path.empty()) { return ""; }; std::string inner_path = FileFinder::MakeCanonical(path, 1); @@ -69,123 +70,141 @@ ZipFilesystem::ZipFilesystem(std::string base_path, FilesystemView parent_fs, St bool is_utf8; encoding = ToString(enc); - if (FindCentralDirectory(zipfile, central_directory_offset, central_directory_size, central_directory_entries)) { - if (encoding.empty()) { - zipfile.seekg(central_directory_offset); - std::stringstream filename_guess; - - // Guess the encoding first - int items = 0; - while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { - // Only consider Non-ASCII & Non-UTF8 for encoding detection - // Skip directories, files already contain the paths - if (is_utf8 || filepath.back() == '/' || Utils::StringIsAscii(filepath)) { - continue; - } - // Codepath will be only entered by Windows "compressed folder" ZIPs (uses local encoding) and - // 7zip (uses CP932 for Western European filenames) - - auto pos = filepath.find_last_of('/'); - if (pos == std::string::npos) { - filename_guess << filepath; - } else { - filename_guess << filepath.substr(pos + 1); - } + if (!FindCentralDirectory(zipfile, central_directory_offset, central_directory_size, central_directory_entries)) { + Output::Debug("ZipFS: {} is not a valid archive", GetPath()); + return; + } - ++items; + if (encoding.empty()) { + zipfile.seekg(central_directory_offset); + std::stringstream filename_guess; - if (items == 10) { - break; - } + // Guess the encoding first + int items = 0; + while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { + // Only consider Non-ASCII & Non-UTF8 for encoding detection + // Skip directories, files already contain the paths + if (is_utf8 || filepath.back() == '/' || Utils::StringIsAscii(filepath)) { + continue; } + // Codepath will be only entered by Windows "compressed folder" ZIPs (uses local encoding) and + // 7zip (uses CP932 for Western European filenames) - if (items == 0) { - // Only ASCII or UTF-8 flags set - encoding = "UTF-8"; + auto pos = filepath.find_last_of('/'); + if (pos == std::string::npos) { + filename_guess << filepath; } else { - std::vector encodings = lcf::ReaderUtil::DetectEncodings(filename_guess.str()); - for (const auto &enc_ : encodings) { - std::string enc_test = lcf::ReaderUtil::Recode("\\", enc_); - if (enc_test.empty()) { - // Bad encoding - Output::Debug("Bad encoding: {}. Trying next.", enc_); - continue; - } - encoding = enc_; - break; - } + filename_guess << filepath.substr(pos + 1); } - Output::Debug("Detected ZIP encoding: {}", encoding); - } - bool enc_is_utf8 = encoding == "UTF-8"; - zipfile.clear(); - zipfile.seekg(central_directory_offset); + ++items; - std::vector paths; - while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { - if (is_utf8 || enc_is_utf8 || Utils::StringIsAscii(filepath)) { - // No reencoding necessary - filepath_cp437.clear(); - } else { - // also store CP437 to ensure files inside 7zip zip archives are found - filepath_cp437 = lcf::ReaderUtil::Recode(filepath, "437"); - filepath = lcf::ReaderUtil::Recode(filepath, encoding); + if (items == 10) { + break; } + } - // Workaround ZIP archives containing invalid "\" paths created by .net or Powershell - std::replace(filepath_cp437.begin(), filepath_cp437.end(), '\\', '/'); - std::replace(filepath.begin(), filepath.end(), '\\', '/'); + if (items == 0) { + // Only ASCII or UTF-8 flags set + encoding = "UTF-8"; + } else { + std::vector encodings = lcf::ReaderUtil::DetectEncodings(filename_guess.str()); + for (const auto &enc_ : encodings) { + lcf::Encoder lcf_encoder(enc_); + if (!lcf_encoder.IsOk()) { + // Bad encoding + Output::Debug("Bad encoding: {}. Trying next.", enc_); + continue; + } + encoding = enc_; + break; + } + } + Output::Debug("Detected ZIP encoding: {}", encoding); + } + bool enc_is_utf8 = encoding == "UTF-8"; + + zipfile.clear(); + zipfile.seekg(central_directory_offset); + + lcf::Encoder detected_encoder(encoding); + lcf::Encoder cp437_encoder("437"); + std::vector paths; + while (ReadCentralDirectoryEntry(zipfile, filepath, entry, is_utf8)) { + if (is_utf8 || enc_is_utf8 || Utils::StringIsAscii(filepath)) { + // No reencoding necessary + filepath_cp437.clear(); + } else { + // also store CP437 to ensure files inside 7zip zip archives are found + filepath_cp437 = filepath; + cp437_encoder.Encode(filepath_cp437); + detected_encoder.Encode(filepath); + } - // check if the entry is an directory or not (indicated by trailing /) - // this will fail when the (game) directory has cp437, but the users can rename it before - if (filepath.back() == '/') { - filepath = filepath.substr(0, filepath.size() - 1); + // Workaround ZIP archives containing invalid "\" paths created by .net or Powershell + std::replace(filepath_cp437.begin(), filepath_cp437.end(), '\\', '/'); + std::replace(filepath.begin(), filepath.end(), '\\', '/'); - // Determine intermediate directories - while (!filepath.empty()) { - paths.push_back(filepath); - filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); - } - } else { - zip_entries.emplace_back(filepath, entry); - if (!filepath_cp437.empty()) { - zip_entries_cp437.emplace_back(filepath_cp437, entry); - } + // check if the entry is an directory or not (indicated by trailing /) + // this will fail when the (game) directory has cp437, but the users can rename it before + if (filepath.back() == '/') { + filepath = filepath.substr(0, filepath.size() - 1); - // Determine intermediate directories - for (;;) { - filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); - if (filepath.empty()) { - break; - } - paths.push_back(filepath); + // Determine intermediate directories + while (!filepath.empty()) { + paths.push_back(filepath); + filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); + } + } else { + zip_entries.emplace_back(filepath, entry); + if (!filepath_cp437.empty()) { + zip_entries_cp437.emplace_back(filepath_cp437, entry); + } + + // Determine intermediate directories + for (;;) { + filepath = std::get<0>(FileFinder::GetPathAndFilename(filepath)); + if (filepath.empty()) { + break; } + paths.push_back(filepath); } } - // Build directories - entry = {}; - entry.is_directory = true; - - // add root path - paths.emplace_back(""); - - std::sort(paths.begin(), paths.end()); - auto del = std::unique(paths.begin(), paths.end()); - paths.erase(del, paths.end()); - for (const auto& e : paths) { - zip_entries.emplace_back(e, entry); - } - - std::sort(zip_entries.begin(), zip_entries.end(), [](auto& a, auto& b) { - return a.first < b.first; - }); - std::sort(zip_entries_cp437.begin(), zip_entries_cp437.end(), [](auto& a, auto& b) { - return a.first < b.first; - }); - } else { - Output::Debug("ZipFS: {} is not a valid archive", GetPath()); } + // Build directories + entry = {}; + entry.is_directory = true; + + // add root path + paths.emplace_back(""); + + std::sort(paths.begin(), paths.end()); + auto del = std::unique(paths.begin(), paths.end()); + paths.erase(del, paths.end()); + for (const auto& e : paths) { + zip_entries.emplace_back(e, entry); + } + + // entries can be duplicated in the archive, e.g. when appending to the archive. + // Use a stable sort to preserve this order. + std::stable_sort(zip_entries.begin(), zip_entries.end(), [](auto& a, auto& b) { + return a.first < b.first; + }); + std::stable_sort(zip_entries_cp437.begin(), zip_entries_cp437.end(), [](auto& a, auto& b) { + return a.first < b.first; + }); + + // Then remove all duplicates but keep the last + // The archive of the game "Steamed Hams" has a file with the same name as a folder. This filtering also helps against this issue. + auto entries_del_it = std::unique(zip_entries.rbegin(), zip_entries.rend(), [](auto& a, auto& b) { + return a.first == b.first; + }); + zip_entries.erase(zip_entries.begin(), entries_del_it.base()); + + entries_del_it = std::unique(zip_entries_cp437.rbegin(), zip_entries_cp437.rend(), [](auto& a, auto& b) { + return a.first == b.first; + }); + zip_entries_cp437.erase(zip_entries_cp437.begin(), entries_del_it.base()); } bool ZipFilesystem::FindCentralDirectory(std::istream& zipfile, uint32_t& offset, uint32_t& size, uint16_t& num_entries) const { From d6d33709025f16ff125a6731e1734bd138b1824f Mon Sep 17 00:00:00 2001 From: Ghabry Date: Fri, 27 Oct 2023 23:11:11 +0200 Subject: [PATCH 5/5] lhasa: Add to README and license window (also remove dirent from it as is not used anymore) --- README.md | 5 +++-- src/window_settings.cpp | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 479833974f..6526058a50 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Documentation is available at the documentation wiki: https://wiki.easyrpg.org - SDL2 for screen backend support. - Pixman for low level pixel manipulation. - libpng for PNG image support. -- zlib for XYZ image support. -- fmtlib for interal logging. +- zlib for XYZ image and ZIP archive support. +- fmtlib for text formatting and interal logging. ### extended / recommended @@ -34,6 +34,7 @@ Documentation is available at the documentation wiki: https://wiki.easyrpg.org - libsndfile for better WAVE audio support. - libxmp for tracker music support. - SpeexDSP or libsamplerate for proper audio resampling. +- lhasa for LHA (.lzh) archive support. SDL 1.2 is still supported, but deprecated. diff --git a/src/window_settings.cpp b/src/window_settings.cpp index a1c2479f83..b4137044ef 100644 --- a/src/window_settings.cpp +++ b/src/window_settings.cpp @@ -356,10 +356,10 @@ void Window_Settings::RefreshLicense() { AddOption(MenuItem("ALSA", "Linux sound support (used for MIDI playback)", "LGPL2.1+"), [](){}); #endif #endif - AddOption(MenuItem("rang", "Colors the terminal output", "Unlicense"), [](){}); -#ifdef _WIN32 - AddOption(MenuItem("dirent", "Dirent interface for Microsoft Visual Studio", "MIT"), [](){}); +#ifdef HAVE_LHASA + AddOption(MenuItem("lhasa", "For parsing LHA (.lzh) archives", "ISC"), [](){}); #endif + AddOption(MenuItem("rang", "Colors the terminal output", "Unlicense"), [](){}); AddOption(MenuItem("Baekmuk", "Korean font family", "Baekmuk"), [](){}); AddOption(MenuItem("Shinonome", "Japanese font family", "Public Domain"), [](){}); AddOption(MenuItem("ttyp0", "ttyp0 font family", "ttyp0"), [](){});