From 480bc8152236d2830f2bd8a46fabcf70f8128382 Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 7 Oct 2023 10:54:09 +0200 Subject: [PATCH 01/14] Extract search path related code from Linker into separate class --- src/Linker/Linker.cpp | 224 +++---------------------------- src/Linker/Linker.h | 14 +- src/Linker/LinkerSearchPaths.cpp | 190 ++++++++++++++++++++++++++ src/Linker/LinkerSearchPaths.h | 46 +++++++ src/Linker/main.cpp | 4 +- 5 files changed, 260 insertions(+), 218 deletions(-) create mode 100644 src/Linker/LinkerSearchPaths.cpp create mode 100644 src/Linker/LinkerSearchPaths.h diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index 87d254224..205b315bd 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -15,6 +15,7 @@ #include "SearchPath/SearchPathFilesystem.h" #include "ObjContainer/IWD/IWD.h" #include "LinkerArgs.h" +#include "LinkerSearchPaths.h" #include "ZoneWriting.h" #include "Game/IW3/ZoneCreatorIW3.h" #include "ZoneCreation/ZoneCreationContext.h" @@ -40,195 +41,16 @@ const IZoneCreator* const ZONE_CREATORS[] new T6::ZoneCreator() }; -class Linker::Impl +class LinkerImpl final : public Linker { static constexpr const char* METADATA_NAME = "name"; static constexpr const char* METADATA_GAME = "game"; static constexpr const char* METADATA_GDT = "gdt"; LinkerArgs m_args; - std::vector> m_loaded_project_search_paths; - SearchPaths m_asset_search_paths; - SearchPaths m_gdt_search_paths; - SearchPaths m_source_search_paths; + LinkerSearchPaths m_search_paths; std::vector> m_loaded_zones; - /** - * \brief Loads a search path. - * \param searchPath The search path to load. - */ - void LoadSearchPath(ISearchPath* searchPath) const - { - if (m_args.m_verbose) - { - printf("Loading search path: \"%s\"\n", searchPath->GetPath().c_str()); - } - - ObjLoading::LoadIWDsInSearchPath(searchPath); - } - - /** - * \brief Unloads a search path. - * \param searchPath The search path to unload. - */ - void UnloadSearchPath(ISearchPath* searchPath) const - { - if (m_args.m_verbose) - { - printf("Unloading search path: \"%s\"\n", searchPath->GetPath().c_str()); - } - - ObjLoading::UnloadIWDsInSearchPath(searchPath); - } - - SearchPaths GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName) - { - SearchPaths searchPathsForProject; - - for (const auto& searchPathStr : m_args.GetAssetSearchPathsForProject(gameName, projectName)) - { - auto absolutePath = fs::absolute(searchPathStr); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; - - auto searchPath = std::make_unique(searchPathStr); - LoadSearchPath(searchPath.get()); - searchPathsForProject.IncludeSearchPath(searchPath.get()); - m_loaded_project_search_paths.emplace_back(std::move(searchPath)); - } - - searchPathsForProject.IncludeSearchPath(&m_asset_search_paths); - - for (auto* iwd : IWD::Repository) - { - searchPathsForProject.IncludeSearchPath(iwd); - } - - return searchPathsForProject; - } - - SearchPaths GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName) - { - SearchPaths searchPathsForProject; - - for (const auto& searchPathStr : m_args.GetGdtSearchPathsForProject(gameName, projectName)) - { - auto absolutePath = fs::absolute(searchPathStr); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding gdt search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; - - searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); - } - - searchPathsForProject.IncludeSearchPath(&m_gdt_search_paths); - - return searchPathsForProject; - } - - SearchPaths GetSourceSearchPathsForProject(const std::string& projectName) - { - SearchPaths searchPathsForProject; - - for (const auto& searchPathStr : m_args.GetSourceSearchPathsForProject(projectName)) - { - auto absolutePath = fs::absolute(searchPathStr); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding source search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding source search path: " << absolutePath.string() << std::endl; - - searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); - } - - searchPathsForProject.IncludeSearchPath(&m_source_search_paths); - - return searchPathsForProject; - } - - /** - * \brief Initializes the Linker object's search paths based on the user's input. - * \return \c true if building the search paths was successful, otherwise \c false. - */ - bool BuildProjectIndependentSearchPaths() - { - for (const auto& path : m_args.GetProjectIndependentAssetSearchPaths()) - { - auto absolutePath = fs::absolute(path); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; - - auto searchPath = std::make_unique(absolutePath.string()); - LoadSearchPath(searchPath.get()); - m_asset_search_paths.CommitSearchPath(std::move(searchPath)); - } - - for (const auto& path : m_args.GetProjectIndependentGdtSearchPaths()) - { - auto absolutePath = fs::absolute(path); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Loading gdt search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; - - m_gdt_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); - } - - for (const auto& path : m_args.GetProjectIndependentSourceSearchPaths()) - { - auto absolutePath = fs::absolute(path); - - if (!fs::is_directory(absolutePath)) - { - if (m_args.m_verbose) - std::cout << "Loading source search path (Not found): " << absolutePath.string() << std::endl; - continue; - } - - if (m_args.m_verbose) - std::cout << "Adding source search path: " << absolutePath.string() << std::endl; - - m_source_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); - } - - return true; - } - bool IncludeAdditionalZoneDefinitions(const std::string& initialFileName, ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const { std::set sourceNames; @@ -500,7 +322,7 @@ class Linker::Impl bool BuildProject(const std::string& projectName) { - auto sourceSearchPaths = GetSourceSearchPathsForProject(projectName); + auto sourceSearchPaths = m_search_paths.GetSourceSearchPathsForProject(projectName); const auto zoneDefinition = ReadZoneDefinition(projectName, &sourceSearchPaths); if (!zoneDefinition) @@ -513,19 +335,15 @@ class Linker::Impl for (auto& c : gameName) c = static_cast(std::tolower(c)); - auto assetSearchPaths = GetAssetSearchPathsForProject(gameName, projectName); - auto gdtSearchPaths = GetGdtSearchPathsForProject(gameName, projectName); + auto assetSearchPaths = m_search_paths.GetAssetSearchPathsForProject(gameName, projectName); + auto gdtSearchPaths = m_search_paths.GetGdtSearchPathsForProject(gameName, projectName); const auto zone = CreateZoneForDefinition(projectName, *zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths); auto result = zone != nullptr; if (zone) result = WriteZoneToFile(projectName, zone.get()); - for (const auto& loadedSearchPath : m_loaded_project_search_paths) - { - UnloadSearchPath(loadedSearchPath.get()); - } - m_loaded_project_search_paths.clear(); + m_search_paths.UnloadProjectSpecificSearchPaths(); return result; } @@ -576,18 +394,17 @@ class Linker::Impl } public: - Impl() - = default; + LinkerImpl() + : m_search_paths(m_args) + { + } - /** - * \copydoc Linker::Start - */ - bool Start(const int argc, const char** argv) + bool Start(const int argc, const char** argv) override { if (!m_args.ParseArgs(argc, argv)) return false; - if (!BuildProjectIndependentSearchPaths()) + if (!m_search_paths.BuildProjectIndependentSearchPaths()) return false; if (!LoadZones()) @@ -609,18 +426,7 @@ class Linker::Impl } }; -Linker::Linker() -{ - m_impl = new Impl(); -} - -Linker::~Linker() -{ - delete m_impl; - m_impl = nullptr; -} - -bool Linker::Start(const int argc, const char** argv) const +std::unique_ptr Linker::Create() { - return m_impl->Start(argc, argv); + return std::make_unique(); } diff --git a/src/Linker/Linker.h b/src/Linker/Linker.h index 3e452c906..06bb0cb04 100644 --- a/src/Linker/Linker.h +++ b/src/Linker/Linker.h @@ -1,13 +1,11 @@ #pragma once +#include class Linker { - class Impl; - Impl* m_impl; - public: - Linker(); - ~Linker(); + Linker() = default; + virtual ~Linker() = default; Linker(const Linker& other) = delete; Linker(Linker&& other) noexcept = delete; @@ -20,5 +18,7 @@ class Linker * \param argv The command line arguments. * \return \c true if the application was successful or \c false if an error occurred. */ - bool Start(int argc, const char** argv) const; -}; \ No newline at end of file + virtual bool Start(int argc, const char** argv) = 0; + + static std::unique_ptr Create(); +}; diff --git a/src/Linker/LinkerSearchPaths.cpp b/src/Linker/LinkerSearchPaths.cpp new file mode 100644 index 000000000..48ce337d6 --- /dev/null +++ b/src/Linker/LinkerSearchPaths.cpp @@ -0,0 +1,190 @@ +#include "LinkerSearchPaths.h" + +#include +#include + +#include "ObjLoading.h" +#include "ObjContainer/IWD/IWD.h" +#include "SearchPath/SearchPathFilesystem.h" + +namespace fs = std::filesystem; + +LinkerSearchPaths::LinkerSearchPaths(const LinkerArgs& args) + : m_args(args) +{ +} + +void LinkerSearchPaths::LoadSearchPath(ISearchPath* searchPath) const +{ + if (m_args.m_verbose) + { + printf("Loading search path: \"%s\"\n", searchPath->GetPath().c_str()); + } + + ObjLoading::LoadIWDsInSearchPath(searchPath); +} + +void LinkerSearchPaths::UnloadSearchPath(ISearchPath* searchPath) const +{ + if (m_args.m_verbose) + { + printf("Unloading search path: \"%s\"\n", searchPath->GetPath().c_str()); + } + + ObjLoading::UnloadIWDsInSearchPath(searchPath); +} + +SearchPaths LinkerSearchPaths::GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName) +{ + SearchPaths searchPathsForProject; + + for (const auto& searchPathStr : m_args.GetAssetSearchPathsForProject(gameName, projectName)) + { + auto absolutePath = fs::absolute(searchPathStr); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; + + auto searchPath = std::make_unique(searchPathStr); + LoadSearchPath(searchPath.get()); + searchPathsForProject.IncludeSearchPath(searchPath.get()); + m_loaded_project_search_paths.emplace_back(std::move(searchPath)); + } + + searchPathsForProject.IncludeSearchPath(&m_asset_search_paths); + + for (auto* iwd : IWD::Repository) + { + searchPathsForProject.IncludeSearchPath(iwd); + } + + return searchPathsForProject; +} + +SearchPaths LinkerSearchPaths::GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName) +{ + SearchPaths searchPathsForProject; + + for (const auto& searchPathStr : m_args.GetGdtSearchPathsForProject(gameName, projectName)) + { + auto absolutePath = fs::absolute(searchPathStr); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding gdt search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; + + searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); + } + + searchPathsForProject.IncludeSearchPath(&m_gdt_search_paths); + + return searchPathsForProject; +} + +SearchPaths LinkerSearchPaths::GetSourceSearchPathsForProject(const std::string& projectName) +{ + SearchPaths searchPathsForProject; + + for (const auto& searchPathStr : m_args.GetSourceSearchPathsForProject(projectName)) + { + auto absolutePath = fs::absolute(searchPathStr); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding source search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding source search path: " << absolutePath.string() << std::endl; + + searchPathsForProject.CommitSearchPath(std::make_unique(searchPathStr)); + } + + searchPathsForProject.IncludeSearchPath(&m_source_search_paths); + + return searchPathsForProject; +} + +bool LinkerSearchPaths::BuildProjectIndependentSearchPaths() +{ + for (const auto& path : m_args.GetProjectIndependentAssetSearchPaths()) + { + auto absolutePath = fs::absolute(path); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Adding asset search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding asset search path: " << absolutePath.string() << std::endl; + + auto searchPath = std::make_unique(absolutePath.string()); + LoadSearchPath(searchPath.get()); + m_asset_search_paths.CommitSearchPath(std::move(searchPath)); + } + + for (const auto& path : m_args.GetProjectIndependentGdtSearchPaths()) + { + auto absolutePath = fs::absolute(path); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Loading gdt search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding gdt search path: " << absolutePath.string() << std::endl; + + m_gdt_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); + } + + for (const auto& path : m_args.GetProjectIndependentSourceSearchPaths()) + { + auto absolutePath = fs::absolute(path); + + if (!fs::is_directory(absolutePath)) + { + if (m_args.m_verbose) + std::cout << "Loading source search path (Not found): " << absolutePath.string() << std::endl; + continue; + } + + if (m_args.m_verbose) + std::cout << "Adding source search path: " << absolutePath.string() << std::endl; + + m_source_search_paths.CommitSearchPath(std::make_unique(absolutePath.string())); + } + + return true; +} + + +void LinkerSearchPaths::UnloadProjectSpecificSearchPaths() +{ + for (const auto& loadedSearchPath : m_loaded_project_search_paths) + { + UnloadSearchPath(loadedSearchPath.get()); + } + + m_loaded_project_search_paths.clear(); +} diff --git a/src/Linker/LinkerSearchPaths.h b/src/Linker/LinkerSearchPaths.h new file mode 100644 index 000000000..ea5825d2c --- /dev/null +++ b/src/Linker/LinkerSearchPaths.h @@ -0,0 +1,46 @@ +#pragma once +#include "SearchPath/SearchPaths.h" + +#include +#include + +#include "LinkerArgs.h" + +class LinkerSearchPaths +{ +public: + explicit LinkerSearchPaths(const LinkerArgs& args); + + /** + * \brief Loads a search path. + * \param searchPath The search path to load. + */ + void LoadSearchPath(ISearchPath* searchPath) const; + + /** + * \brief Unloads a search path. + * \param searchPath The search path to unload. + */ + void UnloadSearchPath(ISearchPath* searchPath) const; + + SearchPaths GetAssetSearchPathsForProject(const std::string& gameName, const std::string& projectName); + + SearchPaths GetGdtSearchPathsForProject(const std::string& gameName, const std::string& projectName); + + SearchPaths GetSourceSearchPathsForProject(const std::string& projectName); + + /** + * \brief Initializes the Linker object's search paths based on the user's input. + * \return \c true if building the search paths was successful, otherwise \c false. + */ + bool BuildProjectIndependentSearchPaths(); + + void UnloadProjectSpecificSearchPaths(); + +private: + const LinkerArgs& m_args; + std::vector> m_loaded_project_search_paths; + SearchPaths m_asset_search_paths; + SearchPaths m_gdt_search_paths; + SearchPaths m_source_search_paths; +}; diff --git a/src/Linker/main.cpp b/src/Linker/main.cpp index 6e78bec12..36f849476 100644 --- a/src/Linker/main.cpp +++ b/src/Linker/main.cpp @@ -2,7 +2,7 @@ int main(const int argc, const char** argv) { - Linker linker; + const auto linker = Linker::Create(); - return linker.Start(argc, argv) ? 0 : 1; + return linker->Start(argc, argv) ? 0 : 1; } From e4fd8543e32cf4e5315fba844157495cff817489 Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 7 Oct 2023 13:29:27 +0200 Subject: [PATCH 02/14] Read project type from zone file --- src/Linker/Linker.cpp | 104 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index 205b315bd..9ddc255e2 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -41,11 +41,26 @@ const IZoneCreator* const ZONE_CREATORS[] new T6::ZoneCreator() }; +enum class ProjectType +{ + FASTFILE, + IPAK, + + MAX +}; + +constexpr const char* PROJECT_TYPE_NAMES[static_cast(ProjectType::MAX)] +{ + "fastfile", + "ipak" +}; + class LinkerImpl final : public Linker { - static constexpr const char* METADATA_NAME = "name"; static constexpr const char* METADATA_GAME = "game"; static constexpr const char* METADATA_GDT = "gdt"; + static constexpr const char* METADATA_NAME = "name"; + static constexpr const char* METADATA_TYPE = "type"; LinkerArgs m_args; LinkerSearchPaths m_search_paths; @@ -219,6 +234,55 @@ class LinkerImpl final : public Linker return true; } + static bool ProjectTypeByName(ProjectType& projectType, const std::string& projectTypeName) + { + for (auto i = 0u; i < static_cast(ProjectType::MAX); i++) + { + if (projectTypeName == PROJECT_TYPE_NAMES[i]) + { + projectType = static_cast(i); + return true; + } + } + + return false; + } + + static bool GetProjectTypeFromZoneDefinition(ProjectType& projectType, const std::string& projectName, const ZoneDefinition& zoneDefinition) + { + auto firstGameEntry = true; + const auto [rangeBegin, rangeEnd] = zoneDefinition.m_metadata_lookup.equal_range(METADATA_TYPE); + for (auto i = rangeBegin; i != rangeEnd; ++i) + { + ProjectType parsedProjectType; + if (!ProjectTypeByName(parsedProjectType, i->second->m_value)) + { + std::cerr << "Not a valid project type: \"" << i->second->m_value << "\"\n"; + return false; + } + + if (firstGameEntry) + { + projectType = parsedProjectType; + firstGameEntry = false; + } + else + { + if (projectType != parsedProjectType) + { + std::cerr << "Conflicting types in project \"" << projectName << "\": " << PROJECT_TYPE_NAMES[static_cast(projectType)] + << " != " << PROJECT_TYPE_NAMES[static_cast(parsedProjectType)] << std::endl; + return false; + } + } + } + + if (firstGameEntry) + projectType = ProjectType::FASTFILE; + + return true; + } + static bool GetGameNameFromZoneDefinition(std::string& gameName, const std::string& projectName, const ZoneDefinition& zoneDefinition) { auto firstGameEntry = true; @@ -320,16 +384,10 @@ class LinkerImpl final : public Linker return true; } - bool BuildProject(const std::string& projectName) + bool BuildFastFile(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& sourceSearchPaths) { - auto sourceSearchPaths = m_search_paths.GetSourceSearchPathsForProject(projectName); - - const auto zoneDefinition = ReadZoneDefinition(projectName, &sourceSearchPaths); - if (!zoneDefinition) - return false; - std::string gameName; - if (!GetGameNameFromZoneDefinition(gameName, projectName, *zoneDefinition)) + if (!GetGameNameFromZoneDefinition(gameName, projectName, zoneDefinition)) return false; for (auto& c : gameName) @@ -338,7 +396,7 @@ class LinkerImpl final : public Linker auto assetSearchPaths = m_search_paths.GetAssetSearchPathsForProject(gameName, projectName); auto gdtSearchPaths = m_search_paths.GetGdtSearchPathsForProject(gameName, projectName); - const auto zone = CreateZoneForDefinition(projectName, *zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths); + const auto zone = CreateZoneForDefinition(projectName, zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths); auto result = zone != nullptr; if (zone) result = WriteZoneToFile(projectName, zone.get()); @@ -348,6 +406,32 @@ class LinkerImpl final : public Linker return result; } + bool BuildProject(const std::string& projectName) + { + auto sourceSearchPaths = m_search_paths.GetSourceSearchPathsForProject(projectName); + + const auto zoneDefinition = ReadZoneDefinition(projectName, &sourceSearchPaths); + if (!zoneDefinition) + return false; + + ProjectType projectType; + if (!GetProjectTypeFromZoneDefinition(projectType, projectName, *zoneDefinition)) + return false; + + switch (projectType) + { + case ProjectType::FASTFILE: + return BuildFastFile(projectName, *zoneDefinition, sourceSearchPaths); + + case ProjectType::IPAK: + default: + assert(false); + break; + } + + return false; + } + bool LoadZones() { for (const auto& zonePath : m_args.m_zones_to_load) From fb5c67b5ced9811ca9329a86c02037ad4e0338cf Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 7 Oct 2023 15:06:41 +0200 Subject: [PATCH 03/14] Add possibility to include asset lists in project definition --- src/Linker/Game/IW3/ZoneCreatorIW3.cpp | 8 +- src/Linker/Game/IW3/ZoneCreatorIW3.h | 4 +- src/Linker/Game/IW4/ZoneCreatorIW4.cpp | 8 +- src/Linker/Game/IW4/ZoneCreatorIW4.h | 4 +- src/Linker/Game/IW5/ZoneCreatorIW5.cpp | 8 +- src/Linker/Game/IW5/ZoneCreatorIW5.h | 4 +- src/Linker/Game/T5/ZoneCreatorT5.cpp | 6 +- src/Linker/Game/T5/ZoneCreatorT5.h | 2 +- src/Linker/Game/T6/ZoneCreatorT6.cpp | 10 +- src/Linker/Game/T6/ZoneCreatorT6.h | 6 +- src/Linker/Linker.cpp | 91 +++++++++++-------- src/Linker/ZoneCreation/ZoneCreationContext.h | 3 +- .../SequenceZoneDefinitionAssetList.cpp | 19 ++++ .../SequenceZoneDefinitionAssetList.h | 14 +++ .../ZoneDefinition/ZoneDefinitionParser.cpp | 2 + src/ZoneCommon/Zone/AssetList/AssetList.h | 7 ++ .../Zone/Definition/ZoneDefinition.cpp | 10 +- .../Zone/Definition/ZoneDefinition.h | 6 +- 18 files changed, 142 insertions(+), 70 deletions(-) create mode 100644 src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp create mode 100644 src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h diff --git a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp index 23d230c2b..1bdd19ff2 100644 --- a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp +++ b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp @@ -22,7 +22,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -32,9 +32,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW3/ZoneCreatorIW3.h b/src/Linker/Game/IW3/ZoneCreatorIW3.h index 37c772629..b1ae2b3df 100644 --- a/src/Linker/Game/IW3/ZoneCreatorIW3.h +++ b/src/Linker/Game/IW3/ZoneCreatorIW3.h @@ -12,8 +12,8 @@ namespace IW3 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp index 3f1751684..5030fbaa6 100644 --- a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp +++ b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp @@ -21,7 +21,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -31,9 +31,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW4/ZoneCreatorIW4.h b/src/Linker/Game/IW4/ZoneCreatorIW4.h index 2906af3df..ef0cbc54b 100644 --- a/src/Linker/Game/IW4/ZoneCreatorIW4.h +++ b/src/Linker/Game/IW4/ZoneCreatorIW4.h @@ -12,8 +12,8 @@ namespace IW4 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp index 9a1b1ac24..51943da4e 100644 --- a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp +++ b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp @@ -21,7 +21,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -31,9 +31,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW5/ZoneCreatorIW5.h b/src/Linker/Game/IW5/ZoneCreatorIW5.h index 7584ebfc5..61f8475df 100644 --- a/src/Linker/Game/IW5/ZoneCreatorIW5.h +++ b/src/Linker/Game/IW5/ZoneCreatorIW5.h @@ -12,8 +12,8 @@ namespace IW5 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/T5/ZoneCreatorT5.cpp b/src/Linker/Game/T5/ZoneCreatorT5.cpp index 7bd314781..117a64c7b 100644 --- a/src/Linker/Game/T5/ZoneCreatorT5.cpp +++ b/src/Linker/Game/T5/ZoneCreatorT5.cpp @@ -32,9 +32,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T5/ZoneCreatorT5.h b/src/Linker/Game/T5/ZoneCreatorT5.h index 0c6760194..986ab8552 100644 --- a/src/Linker/Game/T5/ZoneCreatorT5.h +++ b/src/Linker/Game/T5/ZoneCreatorT5.h @@ -13,7 +13,7 @@ namespace T5 void AddAssetTypeName(asset_type_t assetType, std::string name); static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; public: diff --git a/src/Linker/Game/T6/ZoneCreatorT6.cpp b/src/Linker/Game/T6/ZoneCreatorT6.cpp index 90f5f14d8..cdd923b6a 100644 --- a/src/Linker/Game/T6/ZoneCreatorT6.cpp +++ b/src/Linker/Game/T6/ZoneCreatorT6.cpp @@ -23,7 +23,7 @@ void ZoneCreator::AddAssetTypeName(asset_type_t assetType, std::string name) m_asset_types_by_name.emplace(std::make_pair(std::move(name), assetType)); } -std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) +std::vector ZoneCreator::CreateGdtList(const ZoneCreationContext& context) { std::vector gdtList; gdtList.reserve(context.m_gdt_files.size()); @@ -33,9 +33,9 @@ std::vector ZoneCreator::CreateGdtList(ZoneCreationContext& context) return gdtList; } -bool ZoneCreator::CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const +bool ZoneCreator::CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const { - for (const auto& ignoreEntry : context.m_ignored_assets) + for (const auto& ignoreEntry : context.m_ignored_assets.m_entries) { const auto foundAssetTypeEntry = m_asset_types_by_name.find(ignoreEntry.m_type); if (foundAssetTypeEntry == m_asset_types_by_name.end()) @@ -58,7 +58,7 @@ void ZoneCreator::CreateZoneAssetPools(Zone* zone) const zone->m_pools->InitPoolDynamic(assetType); } -void ZoneCreator::HandleMetadata(Zone* zone, ZoneCreationContext& context) const +void ZoneCreator::HandleMetadata(Zone* zone, const ZoneCreationContext& context) const { std::vector kvpList; @@ -126,7 +126,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T6/ZoneCreatorT6.h b/src/Linker/Game/T6/ZoneCreatorT6.h index ff0bcc7da..8e04d4483 100644 --- a/src/Linker/Game/T6/ZoneCreatorT6.h +++ b/src/Linker/Game/T6/ZoneCreatorT6.h @@ -12,10 +12,10 @@ namespace T6 std::unordered_map m_asset_types_by_name; void AddAssetTypeName(asset_type_t assetType, std::string name); - static std::vector CreateGdtList(ZoneCreationContext& context); - bool CreateIgnoredAssetMap(ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; + static std::vector CreateGdtList(const ZoneCreationContext& context); + bool CreateIgnoredAssetMap(const ZoneCreationContext& context, std::unordered_map& ignoredAssetMap) const; void CreateZoneAssetPools(Zone* zone) const; - void HandleMetadata(Zone* zone, ZoneCreationContext& context) const; + void HandleMetadata(Zone* zone, const ZoneCreationContext& context) const; public: ZoneCreator(); diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index 9ddc255e2..738173a36 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -12,7 +12,6 @@ #include "ObjWriting.h" #include "ObjLoading.h" #include "SearchPath/SearchPaths.h" -#include "SearchPath/SearchPathFilesystem.h" #include "ObjContainer/IWD/IWD.h" #include "LinkerArgs.h" #include "LinkerSearchPaths.h" @@ -115,6 +114,58 @@ class LinkerImpl final : public Linker return true; } + bool ReadAssetList(const std::string& zoneName, AssetList& assetList, ISearchPath* sourceSearchPath) const + { + { + const auto assetListFileName = "assetlist/" + zoneName + ".csv"; + const auto assetListStream = sourceSearchPath->Open(assetListFileName); + + if (assetListStream.IsOpen()) + { + const AssetListInputStream stream(*assetListStream.m_stream); + AssetListEntry entry; + + while (stream.NextEntry(entry)) + { + assetList.m_entries.emplace_back(std::move(entry)); + } + return true; + } + } + + { + const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath); + + if (zoneDefinition) + { + for (const auto& entry : zoneDefinition->m_assets) + { + assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name); + } + return true; + } + } + + return false; + } + + bool IncludeAssetLists(ZoneDefinition& zoneDefinition, ISearchPath* sourceSearchPath) const + { + for (const auto& assetListName : zoneDefinition.m_asset_lists) + { + AssetList assetList; + if (!ReadAssetList(assetListName, assetList, sourceSearchPath)) + { + std::cerr << "Failed to read asset list \"" << assetListName << "\"\n"; + return false; + } + + zoneDefinition.Include(assetList); + } + + return true; + } + static bool GetNameFromZoneDefinition(std::string& name, const std::string& projectName, const ZoneDefinition& zoneDefinition) { auto firstNameEntry = true; @@ -170,42 +221,10 @@ class LinkerImpl final : public Linker if (!IncludeAdditionalZoneDefinitions(projectName, *zoneDefinition, sourceSearchPath)) return nullptr; - return zoneDefinition; - } - - bool ReadAssetList(const std::string& zoneName, std::vector& assetList, ISearchPath* sourceSearchPath) const - { - { - const auto assetListFileName = "assetlist/" + zoneName + ".csv"; - const auto assetListStream = sourceSearchPath->Open(assetListFileName); - - if (assetListStream.IsOpen()) - { - const AssetListInputStream stream(*assetListStream.m_stream); - AssetListEntry entry; - - while (stream.NextEntry(entry)) - { - assetList.emplace_back(std::move(entry)); - } - return true; - } - } - - { - const auto zoneDefinition = ReadZoneDefinition(zoneName, sourceSearchPath); - - if (zoneDefinition) - { - for (const auto& entry : zoneDefinition->m_assets) - { - assetList.emplace_back(entry.m_asset_type, entry.m_asset_name); - } - return true; - } - } + if (!IncludeAssetLists(*zoneDefinition, sourceSearchPath)) + return nullptr; - return false; + return zoneDefinition; } bool ProcessZoneDefinitionIgnores(const std::string& projectName, ZoneCreationContext& context, ISearchPath* sourceSearchPath) const diff --git a/src/Linker/ZoneCreation/ZoneCreationContext.h b/src/Linker/ZoneCreation/ZoneCreationContext.h index 2db20e1d4..a1c7800c1 100644 --- a/src/Linker/ZoneCreation/ZoneCreationContext.h +++ b/src/Linker/ZoneCreation/ZoneCreationContext.h @@ -2,7 +2,6 @@ #include #include #include -#include #include "SearchPath/ISearchPath.h" #include "Obj/Gdt/Gdt.h" @@ -16,7 +15,7 @@ class ZoneCreationContext ISearchPath* m_asset_search_path; ZoneDefinition* m_definition; std::vector> m_gdt_files; - std::vector m_ignored_assets; + AssetList m_ignored_assets; ZoneCreationContext(); ZoneCreationContext(ISearchPath* assetSearchPath, ZoneDefinition* definition); diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp new file mode 100644 index 000000000..39a356251 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.cpp @@ -0,0 +1,19 @@ +#include "SequenceZoneDefinitionAssetList.h" + +#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionMatcherFactory.h" + +SequenceZoneDefinitionAssetList::SequenceZoneDefinitionAssetList() +{ + const ZoneDefinitionMatcherFactory create(this); + + AddMatchers({ + create.Keyword("assetlist"), + create.Char(','), + create.Field().Capture(CAPTURE_ASSET_LIST_NAME) + }); +} + +void SequenceZoneDefinitionAssetList::ProcessMatch(ZoneDefinition* state, SequenceResult& result) const +{ + state->m_asset_lists.emplace_back(result.NextCapture(CAPTURE_ASSET_LIST_NAME).FieldValue()); +} diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h new file mode 100644 index 000000000..f9a336e49 --- /dev/null +++ b/src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionAssetList.h @@ -0,0 +1,14 @@ +#pragma once + +#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h" + +class SequenceZoneDefinitionAssetList final : public ZoneDefinitionParser::sequence_t +{ + static constexpr auto CAPTURE_ASSET_LIST_NAME = 1; + +protected: + void ProcessMatch(ZoneDefinition* state, SequenceResult& result) const override; + +public: + SequenceZoneDefinitionAssetList(); +}; diff --git a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp index ed52f58fc..a6ae9552c 100644 --- a/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp +++ b/src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp @@ -1,5 +1,6 @@ #include "ZoneDefinitionParser.h" +#include "Sequence/SequenceZoneDefinitionAssetList.h" #include "Sequence/SequenceZoneDefinitionEntry.h" #include "Sequence/SequenceZoneDefinitionIgnore.h" #include "Sequence/SequenceZoneDefinitionInclude.h" @@ -16,6 +17,7 @@ const std::vector::seq new SequenceZoneDefinitionMetaData(), new SequenceZoneDefinitionInclude(), new SequenceZoneDefinitionIgnore(), + new SequenceZoneDefinitionAssetList(), new SequenceZoneDefinitionEntry() }); diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.h b/src/ZoneCommon/Zone/AssetList/AssetList.h index 0bd7c0f3f..1f02b329b 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.h +++ b/src/ZoneCommon/Zone/AssetList/AssetList.h @@ -1,5 +1,6 @@ #pragma once #include +#include class AssetListEntry { @@ -10,3 +11,9 @@ class AssetListEntry AssetListEntry(); AssetListEntry(std::string type, std::string name); }; + +class AssetList +{ +public: + std::vector m_entries; +}; \ No newline at end of file diff --git a/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp b/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp index 1c3ed7e2a..0c710d478 100644 --- a/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp +++ b/src/ZoneCommon/Zone/Definition/ZoneDefinition.cpp @@ -29,7 +29,15 @@ void ZoneDefinition::AddMetaData(std::string key, std::string value) m_metadata_lookup.emplace(std::make_pair(metaDataPtr->m_key, metaDataPtr)); } -void ZoneDefinition::Include(ZoneDefinition& definitionToInclude) +void ZoneDefinition::Include(const AssetList& assetListToInclude) +{ + for (const auto& entry : assetListToInclude.m_entries) + { + m_assets.emplace_back(entry.m_type, entry.m_name, false); + } +} + +void ZoneDefinition::Include(const ZoneDefinition& definitionToInclude) { for (const auto& metaData : definitionToInclude.m_metadata) { diff --git a/src/ZoneCommon/Zone/Definition/ZoneDefinition.h b/src/ZoneCommon/Zone/Definition/ZoneDefinition.h index 2a093f8dc..4ec8a915f 100644 --- a/src/ZoneCommon/Zone/Definition/ZoneDefinition.h +++ b/src/ZoneCommon/Zone/Definition/ZoneDefinition.h @@ -5,6 +5,8 @@ #include #include +#include "Zone/AssetList/AssetList.h" + class ZoneDefinitionEntry { public: @@ -33,9 +35,11 @@ class ZoneDefinition std::vector> m_metadata; std::unordered_multimap m_metadata_lookup; std::vector m_includes; + std::vector m_asset_lists; std::vector m_ignores; std::vector m_assets; void AddMetaData(std::string key, std::string value); - void Include(ZoneDefinition& definitionToInclude); + void Include(const AssetList& assetListToInclude); + void Include(const ZoneDefinition& definitionToInclude); }; \ No newline at end of file From 23d0fe1eb08f5ec334d59336954c6a8d7d3947bd Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 7 Oct 2023 15:17:06 +0200 Subject: [PATCH 04/14] Always load/unload project specific zones --- src/Linker/Linker.cpp | 39 ++++++++++++++++++++------------- src/Utils/Utils/StringUtils.cpp | 12 ++++++++++ src/Utils/Utils/StringUtils.h | 3 +++ 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index 738173a36..d94d39d84 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -25,6 +25,7 @@ #include "Game/T6/ZoneCreatorT6.h" #include "Utils/ObjFileStream.h" +#include "Utils/StringUtils.h" #include "Zone/AssetList/AssetList.h" #include "Zone/AssetList/AssetListStream.h" #include "Zone/Definition/ZoneDefinitionStream.h" @@ -403,28 +404,21 @@ class LinkerImpl final : public Linker return true; } - bool BuildFastFile(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& sourceSearchPaths) + bool BuildFastFile(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& gdtSearchPaths, SearchPaths& sourceSearchPaths) const { - std::string gameName; - if (!GetGameNameFromZoneDefinition(gameName, projectName, zoneDefinition)) - return false; - - for (auto& c : gameName) - c = static_cast(std::tolower(c)); - - auto assetSearchPaths = m_search_paths.GetAssetSearchPathsForProject(gameName, projectName); - auto gdtSearchPaths = m_search_paths.GetGdtSearchPathsForProject(gameName, projectName); - const auto zone = CreateZoneForDefinition(projectName, zoneDefinition, &assetSearchPaths, &gdtSearchPaths, &sourceSearchPaths); auto result = zone != nullptr; if (zone) result = WriteZoneToFile(projectName, zone.get()); - m_search_paths.UnloadProjectSpecificSearchPaths(); - return result; } + bool BuildIPak(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) + { + return false; + } + bool BuildProject(const std::string& projectName) { auto sourceSearchPaths = m_search_paths.GetSourceSearchPathsForProject(projectName); @@ -437,18 +431,33 @@ class LinkerImpl final : public Linker if (!GetProjectTypeFromZoneDefinition(projectType, projectName, *zoneDefinition)) return false; + std::string gameName; + if (!GetGameNameFromZoneDefinition(gameName, projectName, *zoneDefinition)) + return false; + utils::MakeStringLowerCase(gameName); + + auto assetSearchPaths = m_search_paths.GetAssetSearchPathsForProject(gameName, projectName); + auto gdtSearchPaths = m_search_paths.GetGdtSearchPathsForProject(gameName, projectName); + + auto result = false; switch (projectType) { case ProjectType::FASTFILE: - return BuildFastFile(projectName, *zoneDefinition, sourceSearchPaths); + result = BuildFastFile(projectName, *zoneDefinition, assetSearchPaths, gdtSearchPaths, sourceSearchPaths); + break; case ProjectType::IPAK: + result = BuildIPak(projectName, *zoneDefinition, assetSearchPaths, sourceSearchPaths); + break; + default: assert(false); break; } - return false; + m_search_paths.UnloadProjectSpecificSearchPaths(); + + return result; } bool LoadZones() diff --git a/src/Utils/Utils/StringUtils.cpp b/src/Utils/Utils/StringUtils.cpp index e7f84d9a5..3edf0c593 100644 --- a/src/Utils/Utils/StringUtils.cpp +++ b/src/Utils/Utils/StringUtils.cpp @@ -88,4 +88,16 @@ namespace utils inEscape = true; } } + + void MakeStringLowerCase(std::string& str) + { + for (auto& c : str) + c = static_cast(tolower(c)); + } + + void MakeStringUpperCase(std::string& str) + { + for (auto& c : str) + c = static_cast(toupper(c)); + } } diff --git a/src/Utils/Utils/StringUtils.h b/src/Utils/Utils/StringUtils.h index 860f8dbb2..d49de6b90 100644 --- a/src/Utils/Utils/StringUtils.h +++ b/src/Utils/Utils/StringUtils.h @@ -11,4 +11,7 @@ namespace utils void EscapeStringForQuotationMarks(std::ostream& stream, const std::string_view& str); std::string UnescapeStringFromQuotationMarks(const std::string_view& str); void UnescapeStringFromQuotationMarks(std::ostream& stream, const std::string_view& str); + + void MakeStringLowerCase(std::string& str); + void MakeStringUpperCase(std::string& str); } From 85143784653503a95be006525955df8772e761cb Mon Sep 17 00:00:00 2001 From: Jan Date: Sat, 7 Oct 2023 19:41:54 +0200 Subject: [PATCH 05/14] Write IPak base skeleton without data --- src/Common/Game/T6/CommonT6.cpp | 10 + src/Common/Game/T6/CommonT6.h | 1 + src/Linker/Game/IW3/ZoneCreatorIW3.cpp | 2 +- src/Linker/Game/IW4/ZoneCreatorIW4.cpp | 2 +- src/Linker/Game/IW5/ZoneCreatorIW5.cpp | 2 +- src/Linker/Game/T5/ZoneCreatorT5.cpp | 2 +- src/Linker/Game/T6/ZoneCreatorT6.cpp | 2 +- src/Linker/Linker.cpp | 37 ++- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 91 ++++--- src/ObjLoading/ObjContainer/IPak/IPak.cpp | 11 +- .../ObjContainer/IPak/IPakWriter.cpp | 241 ++++++++++++++++++ src/ObjWriting/ObjContainer/IPak/IPakWriter.h | 22 ++ src/ZoneCommon/Zone/AssetList/AssetList.cpp | 9 +- src/ZoneCommon/Zone/AssetList/AssetList.h | 3 +- .../Zone/AssetList/AssetListStream.cpp | 13 +- 15 files changed, 388 insertions(+), 60 deletions(-) diff --git a/src/Common/Game/T6/CommonT6.cpp b/src/Common/Game/T6/CommonT6.cpp index 3d1816161..d2ea35779 100644 --- a/src/Common/Game/T6/CommonT6.cpp +++ b/src/Common/Game/T6/CommonT6.cpp @@ -58,6 +58,16 @@ int Common::Com_HashString(const char* str, const int len) return result; } +uint32_t Common::R_HashString(const char* str, uint32_t hash) +{ + for (const auto* pos = str; *pos; pos++) + { + hash = 33 * hash ^ (*pos | 0x20); + } + + return hash; +} + PackedTexCoords Common::Vec2PackTexCoords(const vec2_t* in) { return PackedTexCoords{ Pack32::Vec2PackTexCoords(in->v) }; diff --git a/src/Common/Game/T6/CommonT6.h b/src/Common/Game/T6/CommonT6.h index ad8f38c06..2c63c62e0 100644 --- a/src/Common/Game/T6/CommonT6.h +++ b/src/Common/Game/T6/CommonT6.h @@ -10,6 +10,7 @@ namespace T6 static int Com_HashKey(const char* str, int maxLen); static int Com_HashString(const char* str); static int Com_HashString(const char* str, int len); + static uint32_t R_HashString(const char* str, uint32_t hash); static PackedTexCoords Vec2PackTexCoords(const vec2_t* in); static PackedUnitVec Vec3PackUnitVec(const vec3_t* in); diff --git a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp index 1bdd19ff2..9c6f50128 100644 --- a/src/Linker/Game/IW3/ZoneCreatorIW3.cpp +++ b/src/Linker/Game/IW3/ZoneCreatorIW3.cpp @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp index 5030fbaa6..f09468261 100644 --- a/src/Linker/Game/IW4/ZoneCreatorIW4.cpp +++ b/src/Linker/Game/IW4/ZoneCreatorIW4.cpp @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp index 51943da4e..27d2dfb1d 100644 --- a/src/Linker/Game/IW5/ZoneCreatorIW5.cpp +++ b/src/Linker/Game/IW5/ZoneCreatorIW5.cpp @@ -71,7 +71,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T5/ZoneCreatorT5.cpp b/src/Linker/Game/T5/ZoneCreatorT5.cpp index 117a64c7b..72e682e8d 100644 --- a/src/Linker/Game/T5/ZoneCreatorT5.cpp +++ b/src/Linker/Game/T5/ZoneCreatorT5.cpp @@ -72,7 +72,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Game/T6/ZoneCreatorT6.cpp b/src/Linker/Game/T6/ZoneCreatorT6.cpp index cdd923b6a..a2885949e 100644 --- a/src/Linker/Game/T6/ZoneCreatorT6.cpp +++ b/src/Linker/Game/T6/ZoneCreatorT6.cpp @@ -126,7 +126,7 @@ std::unique_ptr ZoneCreator::CreateZoneForDefinition(ZoneCreationContext& if (!assetEntry.m_is_reference) continue; - context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name); + context.m_ignored_assets.m_entries.emplace_back(assetEntry.m_asset_type, assetEntry.m_asset_name, assetEntry.m_is_reference); } const auto assetLoadingContext = std::make_unique(zone.get(), context.m_asset_search_path, CreateGdtList(context)); diff --git a/src/Linker/Linker.cpp b/src/Linker/Linker.cpp index d94d39d84..d626364c4 100644 --- a/src/Linker/Linker.cpp +++ b/src/Linker/Linker.cpp @@ -23,6 +23,7 @@ #include "Game/IW5/ZoneCreatorIW5.h" #include "Game/T5/ZoneCreatorT5.h" #include "Game/T6/ZoneCreatorT6.h" +#include "ObjContainer/IPak/IPakWriter.h" #include "Utils/ObjFileStream.h" #include "Utils/StringUtils.h" @@ -141,7 +142,7 @@ class LinkerImpl final : public Linker { for (const auto& entry : zoneDefinition->m_assets) { - assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name); + assetList.m_entries.emplace_back(entry.m_asset_type, entry.m_asset_name, entry.m_is_reference); } return true; } @@ -414,9 +415,39 @@ class LinkerImpl final : public Linker return result; } - bool BuildIPak(const std::string& projectName, ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) + bool BuildIPak(const std::string& projectName, const ZoneDefinition& zoneDefinition, SearchPaths& assetSearchPaths, SearchPaths& sourceSearchPaths) const { - return false; + const fs::path ipakFolderPath(m_args.GetOutputFolderPathForProject(projectName)); + auto ipakFilePath(ipakFolderPath); + ipakFilePath.append(zoneDefinition.m_name + ".ipak"); + + fs::create_directories(ipakFolderPath); + + std::ofstream stream(ipakFilePath, std::fstream::out | std::fstream::binary); + if (!stream.is_open()) + return false; + + const auto ipakWriter = IPakWriter::Create(stream, &assetSearchPaths); + for (const auto& assetEntry : zoneDefinition.m_assets) + { + if (assetEntry.m_is_reference) + continue; + + if (assetEntry.m_asset_type == "image") + ipakWriter->AddImage(assetEntry.m_asset_name); + } + + if (!ipakWriter->Write()) + { + std::cout << "Writing ipak failed." << std::endl; + stream.close(); + return false; + } + + std::cout << "Created ipak \"" << ipakFilePath.string() << "\"\n"; + + stream.close(); + return true; } bool BuildProject(const std::string& projectName) diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index ee259f1da..b496d0dba 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -2,65 +2,78 @@ #include -typedef uint32_t IPakHash; +#include "Utils/FileUtils.h" namespace ipak_consts { - static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; - static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + static constexpr uint32_t IPAK_MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I'); + static constexpr uint32_t IPAK_VERSION = 0x50000; + + static constexpr uint32_t IPAK_INDEX_SECTION = 1; + static constexpr uint32_t IPAK_DATA_SECTION = 2; + static constexpr uint32_t IPAK_BRANDING_SECTION = FileUtils::MakeMagic32('M', 'E', 'T', 'A'); + + static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; + static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; } +typedef uint32_t IPakHash; + struct IPakHeader { - uint32_t magic; - uint32_t version; - uint32_t size; - uint32_t sectionCount; + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t sectionCount; }; struct IPakSection { - uint32_t type; - uint32_t offset; - uint32_t size; - uint32_t itemCount; + uint32_t type; + uint32_t offset; + uint32_t size; + uint32_t itemCount; }; union IPakIndexEntryKey { - struct - { - IPakHash dataHash; - IPakHash nameHash; - }; - uint64_t combinedKey; + struct + { + IPakHash dataHash; + IPakHash nameHash; + }; + + uint64_t combinedKey; }; struct IPakIndexEntry { - IPakIndexEntryKey key; - uint32_t offset; - uint32_t size; + IPakIndexEntryKey key; + uint32_t offset; + uint32_t size; }; struct IPakDataBlockHeader { - union - { - uint32_t countAndOffset; - struct - { - uint32_t offset : 24; - uint32_t count : 8; - }; - }; - union - { - uint32_t commands[31]; - struct - { - uint32_t size : 24; - uint32_t compressed : 8; - }_commands[31]; - }; -}; \ No newline at end of file + union + { + uint32_t countAndOffset; + + struct + { + uint32_t offset : 24; + uint32_t count : 8; + }; + }; + + union + { + uint32_t commands[31]; + + struct + { + uint32_t size : 24; + uint32_t compressed : 8; + } _commands[31]; + }; +}; diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index b02f1c4e4..953cc81eb 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -18,9 +18,6 @@ ObjContainerRepository IPak::Repository; class IPak::Impl : public ObjContainerReferenceable { - static const uint32_t MAGIC = FileUtils::MakeMagic32('K', 'A', 'P', 'I'); - static const uint32_t VERSION = 0x50000; - std::string m_path; std::unique_ptr m_stream; @@ -82,11 +79,11 @@ class IPak::Impl : public ObjContainerReferenceable switch (section.type) { - case 1: + case ipak_consts::IPAK_INDEX_SECTION: m_index_section = std::make_unique(section); break; - case 2: + case ipak_consts::IPAK_DATA_SECTION: m_data_section = std::make_unique(section); break; @@ -108,13 +105,13 @@ class IPak::Impl : public ObjContainerReferenceable return false; } - if (header.magic != MAGIC) + if (header.magic != ipak_consts::IPAK_MAGIC) { printf("Invalid ipak magic '0x%x'.\n", header.magic); return false; } - if (header.version != VERSION) + if (header.version != ipak_consts::IPAK_VERSION) { printf("Unsupported ipak version '%u'.\n", header.version); return false; diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index e69de29bb..c29387b3f 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -0,0 +1,241 @@ +#include "IPakWriter.h" + +#include +#include +#include +#include + +#include "Game/T6/CommonT6.h" +#include "Game/T6/GameT6.h" +#include "ObjContainer/IPak/IPakTypes.h" +#include "Utils/Alignment.h" + +class IPakWriterImpl final : public IPakWriter +{ + static constexpr char BRANDING[] = "Created with OAT - OpenAssetTools"; + static constexpr auto SECTION_COUNT = 3; // Index + Data + Branding + + inline static const std::string PAD_DATA = std::string(256, '\xA7'); + +public: + explicit IPakWriterImpl(std::ostream& stream, ISearchPath* assetSearchPath) + : m_stream(stream), + m_asset_search_path(assetSearchPath), + m_current_offset(0), + m_total_size(0), + m_data_section_offset(0), + m_data_section_size(0u), + m_index_section_offset(0), + m_branding_section_offset(0) + { + } + + void AddImage(std::string imageName) override + { + m_images.emplace_back(std::move(imageName)); + } + + void GoTo(const int64_t offset) + { + m_stream.seekp(offset, std::ios::beg); + m_current_offset = offset; + } + + void Write(const void* data, const size_t dataSize) + { + m_stream.write(static_cast(data), dataSize); + m_current_offset += dataSize; + } + + void Pad(const size_t paddingSize) + { + auto paddingSizeLeft = paddingSize; + while (paddingSizeLeft > 0) + { + const auto writeSize = std::min(paddingSizeLeft, PAD_DATA.size()); + Write(PAD_DATA.data(), writeSize); + + paddingSizeLeft -= writeSize; + } + } + + void AlignToChunkWrite() + { + Pad(static_cast(utils::Align(m_current_offset, 0x8000i64) - m_current_offset)); + } + + void WriteHeaderData() + { + GoTo(0); + + const IPakHeader header{ + ipak_consts::IPAK_MAGIC, + ipak_consts::IPAK_VERSION, + static_cast(m_total_size), + SECTION_COUNT + }; + + const IPakSection dataSection{ + ipak_consts::IPAK_DATA_SECTION, + static_cast(m_data_section_offset), + static_cast(m_data_section_size), + static_cast(m_index_entries.size()) + }; + + const IPakSection indexSection{ + ipak_consts::IPAK_INDEX_SECTION, + static_cast(m_index_section_offset), + static_cast(sizeof(IPakIndexEntry) * m_index_entries.size()), + static_cast(m_index_entries.size()) + }; + + const IPakSection brandingSection{ + ipak_consts::IPAK_BRANDING_SECTION, + static_cast(m_branding_section_offset), + std::extent_v, + 1 + }; + + Write(&header, sizeof(header)); + Write(&dataSection, sizeof(dataSection)); + Write(&indexSection, sizeof(indexSection)); + Write(&brandingSection, sizeof(brandingSection)); + } + + static std::string ImageFileName(const std::string& imageName) + { + std::ostringstream ss; + ss << "images/" << imageName << ".iwi"; + + return ss.str(); + } + + std::unique_ptr ReadImageDataFromSearchPath(const std::string& imageName, size_t& imageSize) const + { + const auto fileName = ImageFileName(imageName); + + const auto openFile = m_asset_search_path->Open(fileName); + if (!openFile.IsOpen()) + { + std::cerr << "Could not open image for writing to IPak \"" << fileName << "\"\n"; + return nullptr; + } + + imageSize = static_cast(openFile.m_length); + auto imageData = std::make_unique(imageSize); + openFile.m_stream->read(imageData.get(), imageSize); + + return imageData; + } + + bool WriteImageData(const std::string& imageName) + { + size_t imageSize; + const auto imageData = ReadImageDataFromSearchPath(imageName, imageSize); + if (!imageData) + return false; + + const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0); + const auto dataHash = static_cast(crc32(0u, reinterpret_cast(imageData.get()), imageSize)); + + IPakIndexEntry indexEntry{}; + indexEntry.key.nameHash = nameHash; + indexEntry.key.dataHash = dataHash & 0x1FFFFFFF; + indexEntry.offset = static_cast(m_current_offset - m_data_section_offset); + + size_t writtenImageSize = 0u; + + // TODO: Write image data + + indexEntry.size = writtenImageSize; + m_index_entries.emplace_back(indexEntry); + + m_data_section_size += writtenImageSize; + return true; + } + + bool WriteDataSection() + { + AlignToChunkWrite(); + m_data_section_offset = m_current_offset; + m_data_section_size = 0u; + + m_index_entries.reserve(m_images.size()); + + return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) + { + return WriteImageData(imageName); + }); + } + + static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2) + { + return entry1.key.combinedKey < entry2.key.combinedKey; + } + + void SortIndexSectionEntries() + { + std::sort(m_index_entries.begin(), m_index_entries.end(), CompareIndices); + } + + void WriteIndexSection() + { + AlignToChunkWrite(); + m_index_section_offset = m_current_offset; + + SortIndexSectionEntries(); + + for (const auto& indexEntry : m_index_entries) + Write(&indexEntry, sizeof(indexEntry)); + } + + void WriteBrandingSection() + { + AlignToChunkWrite(); + m_branding_section_offset = m_current_offset; + + Write(BRANDING, std::extent_v); + } + + void WriteFileEnding() + { + AlignToChunkWrite(); + m_total_size = m_current_offset; + } + + bool Write() override + { + // We will write the header and sections later since they need complementary data + GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT); + AlignToChunkWrite(); + + if (!WriteDataSection()) + return false; + + WriteIndexSection(); + WriteBrandingSection(); + WriteFileEnding(); + + WriteHeaderData(); + + return true; + } + +private: + std::ostream& m_stream; + ISearchPath* m_asset_search_path; + std::vector m_images; + + int64_t m_current_offset; + std::vector m_index_entries; + int64_t m_total_size; + int64_t m_data_section_offset; + size_t m_data_section_size; + int64_t m_index_section_offset; + int64_t m_branding_section_offset; +}; + +std::unique_ptr IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath) +{ + return std::make_unique(stream, assetSearchPath); +} diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h index e69de29bb..f7051873c 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h @@ -0,0 +1,22 @@ +#pragma once +#include +#include + +#include "SearchPath/ISearchPath.h" + +class IPakWriter +{ +public: + IPakWriter() = default; + virtual ~IPakWriter() = default; + + IPakWriter(const IPakWriter& other) = default; + IPakWriter(IPakWriter&& other) noexcept = default; + IPakWriter& operator=(const IPakWriter& other) = default; + IPakWriter& operator=(IPakWriter&& other) noexcept = default; + + virtual void AddImage(std::string imageName) = 0; + virtual bool Write() = 0; + + static std::unique_ptr Create(std::ostream& stream, ISearchPath* assetSearchPath); +}; diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.cpp b/src/ZoneCommon/Zone/AssetList/AssetList.cpp index 56bb487ee..05a135623 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.cpp +++ b/src/ZoneCommon/Zone/AssetList/AssetList.cpp @@ -1,10 +1,13 @@ #include "AssetList.h" AssetListEntry::AssetListEntry() -= default; + : m_is_reference(false) +{ +} -AssetListEntry::AssetListEntry(std::string type, std::string name) +AssetListEntry::AssetListEntry(std::string type, std::string name, const bool isReference) : m_type(std::move(type)), - m_name(std::move(name)) + m_name(std::move(name)), + m_is_reference(isReference) { } diff --git a/src/ZoneCommon/Zone/AssetList/AssetList.h b/src/ZoneCommon/Zone/AssetList/AssetList.h index 1f02b329b..186288fd0 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetList.h +++ b/src/ZoneCommon/Zone/AssetList/AssetList.h @@ -7,9 +7,10 @@ class AssetListEntry public: std::string m_type; std::string m_name; + bool m_is_reference; AssetListEntry(); - AssetListEntry(std::string type, std::string name); + AssetListEntry(std::string type, std::string name, bool isReference); }; class AssetList diff --git a/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp b/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp index e1d849542..546ec1695 100644 --- a/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp +++ b/src/ZoneCommon/Zone/AssetList/AssetListStream.cpp @@ -9,7 +9,7 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const { std::vector row; - while(true) + while (true) { if (!m_stream.NextRow(row)) return false; @@ -18,8 +18,17 @@ bool AssetListInputStream::NextEntry(AssetListEntry& entry) const continue; entry.m_type = row[0]; - if (row.size() >= 2) + if (row.size() >= 3 && row[1].empty()) + { + entry.m_name = row[2]; + entry.m_is_reference = true; + } + else + { entry.m_name = row[1]; + entry.m_is_reference = false; + } + return true; } } From 78196cd6f6cd0161c468a9a23202920519c60cc3 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 14:23:26 +0200 Subject: [PATCH 06/14] Improve code documentation of IPakEntryReadStream --- .../ObjContainer/IPak/IPakEntryReadStream.cpp | 37 +++++++++---- .../ObjContainer/IPak/IPakEntryReadStream.h | 54 +++++++++++++++++-- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index f98f100e4..a8301ba9e 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -40,9 +40,11 @@ IPakEntryReadStream::~IPakEntryReadStream() size_t IPakEntryReadStream::ReadChunks(uint8_t* buffer, const int64_t startPos, const size_t chunkCount) const { m_stream_manager_actions->StartReading(); + m_stream.seekg(startPos); m_stream.read(reinterpret_cast(buffer), static_cast(chunkCount) * IPAK_CHUNK_SIZE); const auto readSize = static_cast(m_stream.gcount()); + m_stream_manager_actions->StopReading(); return readSize / IPAK_CHUNK_SIZE; @@ -52,7 +54,6 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch { // Cannot load more than IPAK_CHUNK_COUNT_PER_READ chunks without overflowing the buffer assert(chunkCount <= IPAK_CHUNK_COUNT_PER_READ); - if (chunkCount > IPAK_CHUNK_COUNT_PER_READ) chunkCount = IPAK_CHUNK_COUNT_PER_READ; @@ -68,8 +69,11 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch const auto endPos = startPos + static_cast(chunkCount) * IPAK_CHUNK_SIZE; + // Check whether the start position is already part of the loaded data + // We might be able to reuse previously loaded data if (startPos >= m_buffer_start_pos && startPos < m_buffer_end_pos) { + // Check whether we need to move data from inside the buffer to the start to account for new start if (m_buffer_start_pos != startPos) { const auto moveEnd = endPos < m_buffer_end_pos ? endPos : m_buffer_end_pos; @@ -78,6 +82,7 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch m_buffer_start_pos = startPos; } + // Check whether we need to load additional data that was not previously loaded if (endPos > m_buffer_end_pos) { const auto readChunkCount = ReadChunks(&m_chunk_buffer[m_buffer_end_pos - startPos], m_buffer_end_pos, @@ -92,11 +97,15 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch return true; } + // Check whether the end position is already part of the loaded data if (endPos > m_buffer_start_pos && endPos <= m_buffer_end_pos) { assert(IPAK_CHUNK_SIZE * IPAK_CHUNK_COUNT_PER_READ - static_cast(m_buffer_start_pos - startPos) >= static_cast(endPos - m_buffer_start_pos)); + + // Move data to make sure the end is at the appropriate position to be able to load the missing data in the front memmove(&m_chunk_buffer[m_buffer_start_pos - startPos], m_chunk_buffer, static_cast(endPos - m_buffer_start_pos)); + // We already established that the start of the buffer is not already loaded so we will need to load additional data nonetheless const auto readChunkCount = ReadChunks(m_chunk_buffer, startPos, static_cast(m_buffer_start_pos - startPos) / IPAK_CHUNK_SIZE); @@ -109,6 +118,7 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch return m_buffer_end_pos == endPos; } + // None of the data needed is already loaded -> Load everything and do not reuse any previously loaded data const auto readChunkCount = ReadChunks(m_chunk_buffer, startPos, chunkCount); m_buffer_start_pos = startPos; @@ -117,22 +127,26 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch return chunkCount == readChunkCount; } -bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const +bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const { if (blockHeader->count > 31) { - printf("IPak block has more than 31 commands: %u -> Invalid\n", blockHeader->count); + std::cerr << "IPak block has more than 31 commands: " << blockHeader->count << " -> Invalid\n"; return false; } + + // We expect the current file to be continued where we left off if (blockHeader->offset != m_file_head) { - // A matching offset is only relevant if a command contains data. + // A matching offset is only relevant if a command contains data for (unsigned currentCommand = 0; currentCommand < blockHeader->count; currentCommand++) { + // If compressed is not 0 or 1 it will not be read and therefore it is okay when the offset does not match + // The game uses IPAK_COMMAND_SKIP as value for compressed when it intends to skip the specified amount of data if (blockHeader->_commands[currentCommand].compressed == 0 || blockHeader->_commands[currentCommand].compressed == 1) { - printf("IPak block offset is not the file head: %u != %lld -> Invalid\n", blockHeader->offset, m_file_head); + std::cerr << "IPak block offset is not the file head: " << blockHeader->offset << " != " << m_file_head << " -> Invalid\n"; return false; } } @@ -141,8 +155,7 @@ bool IPakEntryReadStream::ValidateBlockHeader(IPakDataBlockHeader* blockHeader) return true; } -bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, - const size_t blockOffsetInChunk) +bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, const size_t blockOffsetInChunk) { size_t commandsSize = 0; for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) @@ -158,8 +171,7 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHea { if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ) { - printf("IPak block spans over more than %u blocks (%u), which is not supported.\n", - IPAK_CHUNK_COUNT_PER_READ, requiredChunkCount); + std::cerr << "IPak block spans over more than " << IPAK_CHUNK_COUNT_PER_READ << " blocks (" << requiredChunkCount << "), which is not supported.\n"; return false; } @@ -180,7 +192,6 @@ bool IPakEntryReadStream::NextBlock() const auto chunkStartPos = AlignBackwards(m_pos, IPAK_CHUNK_SIZE); const auto blockOffsetInChunk = static_cast(m_pos - chunkStartPos); - const auto sizeLeftToRead = m_entry_size - m_file_head; auto estimatedChunksToRead = AlignForward(m_entry_size - static_cast(m_pos - m_base_pos), IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; @@ -216,7 +227,7 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com if (result != LZO_E_OK) { - printf("Decompressing block with lzo failed: %i!\n", result); + std::cerr << "Decompressing block with lzo failed: " << result << "!\n"; return false; } @@ -225,6 +236,10 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com m_current_command_offset = 0; m_file_head += outputSize; } + else + { + // Do not process data but instead skip specified commandSize + } } else { diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h index f6b13a1c7..c8be8e797 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.h @@ -33,24 +33,68 @@ class IPakEntryReadStream final : public objbuf int64_t m_buffer_start_pos; int64_t m_buffer_end_pos; - template + template static T AlignForward(const T num, const T alignTo) { return (num + alignTo - 1) / alignTo * alignTo; } - template + template static T AlignBackwards(const T num, const T alignTo) { return num / alignTo * alignTo; } + /** + * \brief Reads the specified chunks from disk. + * \param buffer The location to write the loaded data to. Must be able to hold the specified amount of data. + * \param startPos The file offset at which the data to be loaded starts at. + * \param chunkCount The amount of chunks to be loaded. + * \return The amount of chunks that could be successfully loaded. + */ size_t ReadChunks(uint8_t* buffer, int64_t startPos, size_t chunkCount) const; + + /** + * \brief Make sure the loaded chunk buffer window corresponds to the specified parameters and loads additional data if necessary. + * \param startPos The offset in the file that should be the start of the chunk buffer. + * \param chunkCount The amount of chunks that the buffer should hold. Can not exceed IPAK_CHUNK_COUNT_PER_READ. + * \return \c true if the chunk buffer window could successfully be adjusted, \c false otherwise. + */ bool SetChunkBufferWindow(int64_t startPos, size_t chunkCount); - bool ValidateBlockHeader(IPakDataBlockHeader* blockHeader) const; - bool AdjustChunkBufferWindowForBlockHeader(IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); + + /** + * \brief Makes sure the specified block can be safely loaded. + * \param blockHeader The block header to check. + * \return \c true if the block can be safely loaded, \c false otherwise. + */ + bool ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const; + + /** + * \brief Makes sure that the specified block fits inside the loaded chunk buffer window and adjusts the chunk buffer window if necessary. + * \param blockHeader The header of the block that needs to fit inside the loaded chunk buffer window. + * \param blockOffsetInChunk The offset of the block inside the current chunk. + * \return \c true if the chunk buffer window was either already valid or was successfully adjusted to have all block data loaded. \c false otherwise. + */ + bool AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, size_t blockOffsetInChunk); + + /** + * \brief Ensures the next valid block is loaded. + * \return \c true if a new block was loaded or \c false if no further valid block could be loaded. + */ bool NextBlock(); + + /** + * \brief Processes a command with the specified parameters at the current position. + * \param commandSize The size of the command data + * \param compressed The compression value of the command. Can be \c 0 for uncompressed or \c 1 for lzo compression. Any other value skips the specified size of data. + * \return \c true if the specified command could be correctly processed or \c otherwise. + */ bool ProcessCommand(size_t commandSize, int compressed); + + /** + * \brief Ensures that the next valid command is loaded. + * \return \c true if a new command was loaded or \c false if no further valid command could be loaded. + */ bool AdvanceStream(); public: @@ -67,4 +111,4 @@ class IPakEntryReadStream final : public objbuf std::streamsize xsgetn(char* ptr, std::streamsize count) override; pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode mode) override; pos_type seekpos(pos_type pos, std::ios_base::openmode mode) override; -}; \ No newline at end of file +}; From 2d0ef40335e2cea35813e20b3c6a0ba10a9fa660 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 14:50:14 +0200 Subject: [PATCH 07/14] Remove unused IPakBlock non-bits fields --- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 39 ++++++++++--------- .../ObjContainer/IPak/IPakEntryReadStream.cpp | 24 ++++++------ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index b496d0dba..94982262b 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -15,6 +15,8 @@ namespace ipak_consts static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + + static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF; } typedef uint32_t IPakHash; @@ -53,27 +55,26 @@ struct IPakIndexEntry uint32_t size; }; -struct IPakDataBlockHeader +struct IPakDataBlockCountAndOffset { - union - { - uint32_t countAndOffset; + uint32_t offset : 24; + uint32_t count : 8; +}; - struct - { - uint32_t offset : 24; - uint32_t count : 8; - }; - }; +static_assert(sizeof(IPakDataBlockCountAndOffset) == 4); - union - { - uint32_t commands[31]; +struct IPakDataBlockCommand +{ + uint32_t size : 24; + uint32_t compressed : 8; +}; - struct - { - uint32_t size : 24; - uint32_t compressed : 8; - } _commands[31]; - }; +static_assert(sizeof(IPakDataBlockCommand) == 4); + +struct IPakDataBlockHeader +{ + IPakDataBlockCountAndOffset countAndOffset; + IPakDataBlockCommand commands[31]; }; + +static_assert(sizeof(IPakDataBlockHeader) == 128); diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index a8301ba9e..8da1593ec 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -129,24 +129,24 @@ bool IPakEntryReadStream::SetChunkBufferWindow(const int64_t startPos, size_t ch bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHeader) const { - if (blockHeader->count > 31) + if (blockHeader->countAndOffset.count > 31) { - std::cerr << "IPak block has more than 31 commands: " << blockHeader->count << " -> Invalid\n"; + std::cerr << "IPak block has more than 31 commands: " << blockHeader->countAndOffset.count << " -> Invalid\n"; return false; } // We expect the current file to be continued where we left off - if (blockHeader->offset != m_file_head) + if (blockHeader->countAndOffset.offset != m_file_head) { // A matching offset is only relevant if a command contains data - for (unsigned currentCommand = 0; currentCommand < blockHeader->count; currentCommand++) + for (unsigned currentCommand = 0; currentCommand < blockHeader->countAndOffset.count; currentCommand++) { // If compressed is not 0 or 1 it will not be read and therefore it is okay when the offset does not match // The game uses IPAK_COMMAND_SKIP as value for compressed when it intends to skip the specified amount of data - if (blockHeader->_commands[currentCommand].compressed == 0 - || blockHeader->_commands[currentCommand].compressed == 1) + if (blockHeader->commands[currentCommand].compressed == 0 + || blockHeader->commands[currentCommand].compressed == 1) { - std::cerr << "IPak block offset is not the file head: " << blockHeader->offset << " != " << m_file_head << " -> Invalid\n"; + std::cerr << "IPak block offset is not the file head: " << blockHeader->countAndOffset.offset << " != " << m_file_head << " -> Invalid\n"; return false; } } @@ -158,9 +158,9 @@ bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHe bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBlockHeader* blockHeader, const size_t blockOffsetInChunk) { size_t commandsSize = 0; - for (unsigned commandIndex = 0; commandIndex < blockHeader->count; commandIndex++) + for (unsigned commandIndex = 0; commandIndex < blockHeader->countAndOffset.count; commandIndex++) { - commandsSize += blockHeader->_commands[commandIndex].size; + commandsSize += blockHeader->commands[commandIndex].size; } const size_t requiredChunkCount = AlignForward(blockOffsetInChunk + sizeof(IPakDataBlockHeader) + commandsSize, IPAK_CHUNK_SIZE) / IPAK_CHUNK_SIZE; @@ -255,14 +255,14 @@ bool IPakEntryReadStream::ProcessCommand(const size_t commandSize, const int com bool IPakEntryReadStream::AdvanceStream() { - if (m_current_block == nullptr || m_next_command >= m_current_block->count) + if (m_current_block == nullptr || m_next_command >= m_current_block->countAndOffset.count) { if (!NextBlock()) return false; } - ProcessCommand(m_current_block->_commands[m_next_command].size, - m_current_block->_commands[m_next_command].compressed); + ProcessCommand(m_current_block->commands[m_next_command].size, + m_current_block->commands[m_next_command].compressed); m_next_command++; return true; From abbb697d7c61c19307b60635e59b02fe1207ff72 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 17:54:10 +0200 Subject: [PATCH 08/14] Add data writing for IPaks --- src/ObjCommon/ObjContainer/IPak/IPakTypes.h | 5 + src/ObjLoading/ObjContainer/IPak/IPak.cpp | 6 +- .../ObjContainer/IPak/IPakWriter.cpp | 152 ++++++++++++++++-- src/ObjWriting/ObjContainer/IPak/IPakWriter.h | 2 + src/Utils/Utils/Alignment.h | 10 +- 5 files changed, 157 insertions(+), 18 deletions(-) diff --git a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h index 94982262b..3d516821a 100644 --- a/src/ObjCommon/ObjContainer/IPak/IPakTypes.h +++ b/src/ObjCommon/ObjContainer/IPak/IPakTypes.h @@ -16,7 +16,12 @@ namespace ipak_consts static constexpr size_t IPAK_CHUNK_SIZE = 0x8000; static constexpr size_t IPAK_CHUNK_COUNT_PER_READ = 0x8; + static constexpr uint32_t IPAK_COMMAND_DEFAULT_SIZE = 0x7F00; + static constexpr uint32_t IPAK_COMMAND_UNCOMPRESSED = 0; + static constexpr uint32_t IPAK_COMMAND_COMPRESSED = 1; static constexpr uint32_t IPAK_COMMAND_SKIP = 0xCF; + + static_assert(IPAK_COMMAND_DEFAULT_SIZE <= IPAK_CHUNK_SIZE); } typedef uint32_t IPakHash; diff --git a/src/ObjLoading/ObjContainer/IPak/IPak.cpp b/src/ObjLoading/ObjContainer/IPak/IPak.cpp index 953cc81eb..c8d6fff16 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPak.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPak.cpp @@ -70,8 +70,8 @@ class IPak::Impl : public ObjContainerReferenceable { IPakSection section{}; - m_stream->read(reinterpret_cast(§ion), sizeof section); - if (m_stream->gcount() != sizeof section) + m_stream->read(reinterpret_cast(§ion), sizeof(section)); + if (m_stream->gcount() != sizeof(section)) { printf("Unexpected eof when trying to load section.\n"); return false; @@ -98,7 +98,7 @@ class IPak::Impl : public ObjContainerReferenceable { IPakHeader header{}; - m_stream->read(reinterpret_cast(&header), sizeof header); + m_stream->read(reinterpret_cast(&header), sizeof(header)); if (m_stream->gcount() != sizeof header) { printf("Unexpected eof when trying to load header.\n"); diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index c29387b3f..e401d27bd 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -26,8 +27,14 @@ class IPakWriterImpl final : public IPakWriter m_data_section_offset(0), m_data_section_size(0u), m_index_section_offset(0), - m_branding_section_offset(0) + m_branding_section_offset(0), + m_file_offset(0u), + m_chunk_buffer_window_start(0), + m_current_block{}, + m_current_block_header_offset(0) { + m_decompressed_buffer = std::make_unique(ipak_consts::IPAK_CHUNK_SIZE); + m_lzo_work_buffer = std::make_unique(LZO1X_1_MEM_COMPRESS); } void AddImage(std::string imageName) override @@ -59,9 +66,14 @@ class IPakWriterImpl final : public IPakWriter } } - void AlignToChunkWrite() + void AlignToChunk() { - Pad(static_cast(utils::Align(m_current_offset, 0x8000i64) - m_current_offset)); + Pad(static_cast(utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset)); + } + + void AlignToBlockHeader() + { + Pad(static_cast(utils::Align(m_current_offset, static_cast(sizeof(IPakDataBlockHeader))) - m_current_offset)); } void WriteHeaderData() @@ -128,6 +140,106 @@ class IPakWriterImpl final : public IPakWriter return imageData; } + void FlushBlock() + { + if (m_current_block_header_offset > 0) + { + const auto previousOffset = m_current_offset; + + GoTo(m_current_block_header_offset); + Write(&m_current_block, sizeof(m_current_block)); + GoTo(previousOffset); + } + } + + void FlushChunk() + { + FlushBlock(); + AlignToBlockHeader(); + + const auto nextChunkOffset = utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); + const auto sizeToSkip = static_cast(nextChunkOffset - m_current_offset); + + if (sizeToSkip >= sizeof(IPakDataBlockHeader)) + { + IPakDataBlockHeader skipBlockHeader{}; + skipBlockHeader.countAndOffset.count = 1; + skipBlockHeader.commands[0].compressed = ipak_consts::IPAK_COMMAND_SKIP; + skipBlockHeader.commands[0].size = sizeToSkip - sizeof(IPakDataBlockHeader); + Write(&skipBlockHeader, sizeof(skipBlockHeader)); + } + + AlignToChunk(); + m_chunk_buffer_window_start = m_current_offset; + } + + void StartNewBlock() + { + AlignToBlockHeader(); + m_current_block_header_offset = m_current_offset; + m_current_block = {}; + m_current_block.countAndOffset.offset = static_cast(m_file_offset); + } + + void WriteChunkData(const void* data, const size_t dataSize) + { + auto dataOffset = 0u; + while (dataOffset < dataSize) + { + if (m_current_block.countAndOffset.count >= std::extent_v) + { + FlushBlock(); + StartNewBlock(); + } + + const auto remainingSize = dataSize - dataOffset; + const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE) + - static_cast(m_current_offset - m_chunk_buffer_window_start), 0u); + const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize); + + auto writeUncompressed = true; + if (USE_COMPRESSION) + { + auto outLen = static_cast(ipak_consts::IPAK_CHUNK_SIZE); + const auto result = lzo1x_1_compress(&static_cast(data)[dataOffset], commandSize, reinterpret_cast(m_decompressed_buffer.get()), &outLen, + m_lzo_work_buffer.get()); + + if (result == LZO_E_OK && outLen < commandSize) + { + writeUncompressed = false; + Write(m_decompressed_buffer.get(), outLen); + + const auto currentCommand = m_current_block.countAndOffset.count; + m_current_block.commands[currentCommand].size = static_cast(outLen); + m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_COMPRESSED; + m_current_block.countAndOffset.count = currentCommand + 1u; + } + } + + if (writeUncompressed) + { + Write(data, dataSize); + + const auto currentCommand = m_current_block.countAndOffset.count; + m_current_block.commands[currentCommand].size = commandSize; + m_current_block.commands[currentCommand].compressed = ipak_consts::IPAK_COMMAND_UNCOMPRESSED; + m_current_block.countAndOffset.count = currentCommand + 1u; + } + + dataOffset += commandSize; + } + + m_file_offset += dataSize; + } + + void StartNewFile() + { + FlushBlock(); + StartNewBlock(); + m_file_offset = 0u; + m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); + } + bool WriteImageData(const std::string& imageName) { size_t imageSize; @@ -138,34 +250,39 @@ class IPakWriterImpl final : public IPakWriter const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0); const auto dataHash = static_cast(crc32(0u, reinterpret_cast(imageData.get()), imageSize)); - IPakIndexEntry indexEntry{}; + IPakIndexEntry indexEntry; indexEntry.key.nameHash = nameHash; indexEntry.key.dataHash = dataHash & 0x1FFFFFFF; indexEntry.offset = static_cast(m_current_offset - m_data_section_offset); - size_t writtenImageSize = 0u; - - // TODO: Write image data + StartNewFile(); + const auto startOffset = m_current_offset; + WriteChunkData(imageData.get(), imageSize); + const auto writtenImageSize = static_cast(m_current_offset - startOffset); indexEntry.size = writtenImageSize; m_index_entries.emplace_back(indexEntry); - m_data_section_size += writtenImageSize; return true; } bool WriteDataSection() { - AlignToChunkWrite(); + AlignToChunk(); m_data_section_offset = m_current_offset; m_data_section_size = 0u; m_index_entries.reserve(m_images.size()); - return std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) + const auto result = std::all_of(m_images.begin(), m_images.end(), [this](const std::string& imageName) { return WriteImageData(imageName); }); + + FlushBlock(); + m_data_section_size = static_cast(m_current_offset - m_data_section_offset); + + return result; } static bool CompareIndices(const IPakIndexEntry& entry1, const IPakIndexEntry& entry2) @@ -180,7 +297,7 @@ class IPakWriterImpl final : public IPakWriter void WriteIndexSection() { - AlignToChunkWrite(); + AlignToChunk(); m_index_section_offset = m_current_offset; SortIndexSectionEntries(); @@ -191,7 +308,7 @@ class IPakWriterImpl final : public IPakWriter void WriteBrandingSection() { - AlignToChunkWrite(); + AlignToChunk(); m_branding_section_offset = m_current_offset; Write(BRANDING, std::extent_v); @@ -199,7 +316,7 @@ class IPakWriterImpl final : public IPakWriter void WriteFileEnding() { - AlignToChunkWrite(); + AlignToChunk(); m_total_size = m_current_offset; } @@ -207,7 +324,7 @@ class IPakWriterImpl final : public IPakWriter { // We will write the header and sections later since they need complementary data GoTo(sizeof(IPakHeader) + sizeof(IPakSection) * SECTION_COUNT); - AlignToChunkWrite(); + AlignToChunk(); if (!WriteDataSection()) return false; @@ -233,6 +350,13 @@ class IPakWriterImpl final : public IPakWriter size_t m_data_section_size; int64_t m_index_section_offset; int64_t m_branding_section_offset; + + std::unique_ptr m_decompressed_buffer; + std::unique_ptr m_lzo_work_buffer; + size_t m_file_offset; + int64_t m_chunk_buffer_window_start; + IPakDataBlockHeader m_current_block; + int64_t m_current_block_header_offset; }; std::unique_ptr IPakWriter::Create(std::ostream& stream, ISearchPath* assetSearchPath) diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h index f7051873c..6b9f0a3a0 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.h +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.h @@ -7,6 +7,8 @@ class IPakWriter { public: + static constexpr auto USE_COMPRESSION = true; + IPakWriter() = default; virtual ~IPakWriter() = default; diff --git a/src/Utils/Utils/Alignment.h b/src/Utils/Utils/Alignment.h index 9b0ba20d1..053a46364 100644 --- a/src/Utils/Utils/Alignment.h +++ b/src/Utils/Utils/Alignment.h @@ -3,10 +3,18 @@ namespace utils { template - constexpr T Align(T value, T toNext) + constexpr T Align(const T value, const T toNext) { if (toNext > 0) return (value + toNext - 1) / toNext * toNext; return value; } + + template + constexpr T AlignToPrevious(const T value, const T toPrevious) + { + if (toPrevious > 0) + return value / toPrevious * toPrevious; + return value; + } } From e16ea9de8446cee5679802fabbf2746d139982ed Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 19:08:27 +0200 Subject: [PATCH 09/14] Load T6 GfxImages from raw when building --- src/Common/Game/T6/T6_Assets.h | 4 +- .../T6/AssetLoaders/AssetLoaderGfxImage.cpp | 71 +++++++++++++++++++ .../T6/AssetLoaders/AssetLoaderGfxImage.h | 16 +++++ src/ObjLoading/Game/T6/ObjLoaderT6.cpp | 3 +- 4 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp create mode 100644 src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h diff --git a/src/Common/Game/T6/T6_Assets.h b/src/Common/Game/T6/T6_Assets.h index e41a57026..0f839ca8a 100644 --- a/src/Common/Game/T6/T6_Assets.h +++ b/src/Common/Game/T6/T6_Assets.h @@ -765,10 +765,10 @@ namespace T6 int platform[2]; }; - struct GfxStreamedPartInfo { - unsigned int levelCountAndSize; + uint32_t levelCount : 4; + uint32_t levelSize : 28; unsigned int hash; uint16_t width; uint16_t height; diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp new file mode 100644 index 000000000..dcd4ff821 --- /dev/null +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.cpp @@ -0,0 +1,71 @@ +#include "AssetLoaderGfxImage.h" + +#include +#include +#include +#include + +#include "Game/T6/CommonT6.h" +#include "Game/T6/T6.h" +#include "Image/IwiLoader.h" +#include "Pool/GlobalAssetPool.h" + +using namespace T6; + +void* AssetLoaderGfxImage::CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) +{ + auto* image = memory->Create(); + memset(image, 0, sizeof(GfxImage)); + image->name = memory->Dup(assetName.c_str()); + return image; +} + +bool AssetLoaderGfxImage::CanLoadFromRaw() const +{ + return true; +} + +bool AssetLoaderGfxImage::LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const +{ + const auto fileName = "images/" + assetName + ".iwi"; + const auto file = searchPath->Open(fileName); + if (!file.IsOpen()) + return false; + + const auto fileSize = static_cast(file.m_length); + const auto fileData = std::make_unique(fileSize); + file.m_stream->read(fileData.get(), fileSize); + const auto dataHash = static_cast(crc32(0u, reinterpret_cast(fileData.get()), fileSize)); + + MemoryManager tempMemory; + IwiLoader iwiLoader(&tempMemory); + std::istringstream ss(std::string(fileData.get(), fileSize)); + const auto texture = iwiLoader.LoadIwi(ss); + if (!texture) + { + std::cerr << "Failed to load texture from: " << fileName << "\n"; + return false; + } + + auto* image = memory->Create(); + memset(image, 0, sizeof(GfxImage)); + + image->name = memory->Dup(assetName.c_str()); + image->hash = Common::R_HashString(image->name, 0); + image->delayLoadPixels = true; + + image->noPicmip = !texture->HasMipMaps(); + image->width = static_cast(texture->GetWidth()); + image->height = static_cast(texture->GetHeight()); + image->depth = static_cast(texture->GetDepth()); + + image->streaming = 1; + image->streamedParts[0].levelCount = 1; + image->streamedParts[0].levelSize = static_cast(fileSize); + image->streamedParts[0].hash = dataHash & 0x1FFFFFFF; + image->streamedPartCount = 1; + + manager->AddAsset(ASSET_TYPE_IMAGE, assetName, image); + + return true; +} diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h new file mode 100644 index 000000000..f88f76853 --- /dev/null +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderGfxImage.h @@ -0,0 +1,16 @@ +#pragma once +#include "Game/T6/T6.h" +#include "AssetLoading/BasicAssetLoader.h" +#include "AssetLoading/IAssetLoadingManager.h" +#include "SearchPath/ISearchPath.h" + +namespace T6 +{ + class AssetLoaderGfxImage final : public BasicAssetLoader + { + public: + _NODISCARD void* CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) override; + _NODISCARD bool CanLoadFromRaw() const override; + bool LoadFromRaw(const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const override; + }; +} diff --git a/src/ObjLoading/Game/T6/ObjLoaderT6.cpp b/src/ObjLoading/Game/T6/ObjLoaderT6.cpp index 13b385e78..bafa90bb5 100644 --- a/src/ObjLoading/Game/T6/ObjLoaderT6.cpp +++ b/src/ObjLoading/Game/T6/ObjLoaderT6.cpp @@ -7,6 +7,7 @@ #include "ObjContainer/IPak/IPak.h" #include "ObjLoading.h" #include "AssetLoaders/AssetLoaderFontIcon.h" +#include "AssetLoaders/AssetLoaderGfxImage.h" #include "AssetLoaders/AssetLoaderLocalizeEntry.h" #include "AssetLoaders/AssetLoaderPhysConstraints.h" #include "AssetLoaders/AssetLoaderPhysPreset.h" @@ -45,7 +46,7 @@ namespace T6 REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_XMODEL, XModel)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_MATERIAL, Material)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_TECHNIQUE_SET, MaterialTechniqueSet)) - REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_IMAGE, GfxImage)) + REGISTER_ASSET_LOADER(AssetLoaderGfxImage) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND, SndBank)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_SOUND_PATCH, SndPatch)) REGISTER_ASSET_LOADER(BASIC_LOADER(ASSET_TYPE_CLIPMAP, clipMap_t)) From 9b35b90a28b37a20c7fce0fc42d8d9753fcd2649 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 21:54:38 +0200 Subject: [PATCH 10/14] Fix IPak reading error message having incorrect text --- src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp index 8da1593ec..f3b5a1edd 100644 --- a/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp +++ b/src/ObjLoading/ObjContainer/IPak/IPakEntryReadStream.cpp @@ -146,7 +146,7 @@ bool IPakEntryReadStream::ValidateBlockHeader(const IPakDataBlockHeader* blockHe if (blockHeader->commands[currentCommand].compressed == 0 || blockHeader->commands[currentCommand].compressed == 1) { - std::cerr << "IPak block offset is not the file head: " << blockHeader->countAndOffset.offset << " != " << m_file_head << " -> Invalid\n"; + std::cerr << "IPak block offset (" << blockHeader->countAndOffset.offset << ") is not the file head (" << m_file_head << ") -> Invalid\n"; return false; } } @@ -171,7 +171,7 @@ bool IPakEntryReadStream::AdjustChunkBufferWindowForBlockHeader(const IPakDataBl { if (requiredChunkCount > IPAK_CHUNK_COUNT_PER_READ) { - std::cerr << "IPak block spans over more than " << IPAK_CHUNK_COUNT_PER_READ << " blocks (" << requiredChunkCount << "), which is not supported.\n"; + std::cerr << "IPak block spans over more than " << IPAK_CHUNK_COUNT_PER_READ << " chunks (" << requiredChunkCount << "), which is not supported.\n"; return false; } From d779dfd884462d6d126dc22f9a6284ad520834a6 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 22:12:25 +0200 Subject: [PATCH 11/14] Fix writing block header incorrectly --- src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index e401d27bd..60c7aa3e5 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -179,6 +179,9 @@ class IPakWriterImpl final : public IPakWriter m_current_block_header_offset = m_current_offset; m_current_block = {}; m_current_block.countAndOffset.offset = static_cast(m_file_offset); + + // Reserve space to later write actual block header data + GoTo(m_current_offset + sizeof(IPakDataBlockHeader)); } void WriteChunkData(const void* data, const size_t dataSize) @@ -227,16 +230,16 @@ class IPakWriterImpl final : public IPakWriter } dataOffset += commandSize; + m_file_offset += commandSize; } - - m_file_offset += dataSize; } void StartNewFile() { FlushBlock(); - StartNewBlock(); + m_file_offset = 0u; + StartNewBlock(); m_chunk_buffer_window_start = utils::AlignToPrevious(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)); } @@ -250,13 +253,14 @@ class IPakWriterImpl final : public IPakWriter const auto nameHash = T6::Common::R_HashString(imageName.c_str(), 0); const auto dataHash = static_cast(crc32(0u, reinterpret_cast(imageData.get()), imageSize)); + StartNewFile(); + const auto startOffset = m_current_block_header_offset; + IPakIndexEntry indexEntry; indexEntry.key.nameHash = nameHash; indexEntry.key.dataHash = dataHash & 0x1FFFFFFF; - indexEntry.offset = static_cast(m_current_offset - m_data_section_offset); + indexEntry.offset = static_cast(startOffset - m_data_section_offset); - StartNewFile(); - const auto startOffset = m_current_offset; WriteChunkData(imageData.get(), imageSize); const auto writtenImageSize = static_cast(m_current_offset - startOffset); From fa72a6d5db408d944e44b96bfa2a0059f5782af6 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 22:12:42 +0200 Subject: [PATCH 12/14] Fix writing whole file everytime an uncompressed command is written --- src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index 60c7aa3e5..fd081f4cc 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -221,7 +221,7 @@ class IPakWriterImpl final : public IPakWriter if (writeUncompressed) { - Write(data, dataSize); + Write(&static_cast(data)[dataOffset], commandSize); const auto currentCommand = m_current_block.countAndOffset.count; m_current_block.commands[currentCommand].size = commandSize; From 75cd9bd4e49047af761bdb3453c84767e1787170 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 22:31:21 +0200 Subject: [PATCH 13/14] Fix not respecting chunk buffer window border --- src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index fd081f4cc..2dedce49a 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -198,6 +198,14 @@ class IPakWriterImpl final : public IPakWriter const auto remainingSize = dataSize - dataOffset; const auto remainingChunkBufferWindowSize = std::max((ipak_consts::IPAK_CHUNK_COUNT_PER_READ * ipak_consts::IPAK_CHUNK_SIZE) - static_cast(m_current_offset - m_chunk_buffer_window_start), 0u); + + if (remainingChunkBufferWindowSize == 0) + { + FlushChunk(); + StartNewBlock(); + continue; + } + const auto commandSize = std::min(std::min(remainingSize, ipak_consts::IPAK_COMMAND_DEFAULT_SIZE), remainingChunkBufferWindowSize); auto writeUncompressed = true; From 4811d41ec8a1de005ad7a6fb2b5d338d0890c038 Mon Sep 17 00:00:00 2001 From: Jan Date: Sun, 15 Oct 2023 22:31:50 +0200 Subject: [PATCH 14/14] Fix calculating wrong chunk buffer window when only the block header is inside a chunk and all the data is in the next --- src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp index 2dedce49a..6767a64d4 100644 --- a/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp +++ b/src/ObjWriting/ObjContainer/IPak/IPakWriter.cpp @@ -176,6 +176,11 @@ class IPakWriterImpl final : public IPakWriter void StartNewBlock() { AlignToBlockHeader(); + + // Skip to the next chunk when only the header could fit into the current chunk anyway + if (static_cast(utils::Align(m_current_offset, static_cast(ipak_consts::IPAK_CHUNK_SIZE)) - m_current_offset) <= sizeof(IPakDataBlockHeader)) + FlushChunk(); + m_current_block_header_offset = m_current_offset; m_current_block = {}; m_current_block.countAndOffset.offset = static_cast(m_file_offset);