-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
reworked extract & address some thread safety (#25)
* moved to libarchive * slowed down ui refresh & moved to async * used atomic types for shared progress * update Image to use unique_ptr
- Loading branch information
Showing
13 changed files
with
210 additions
and
171 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,126 +1,166 @@ | ||
| #include "util/extract.hpp" | ||
|
|
||
| #include <dirent.h> | ||
| #include <minizip/unzip.h> | ||
|
|
||
| #include <string> | ||
| #include <vector> | ||
| #include <borealis.hpp> | ||
| #include <strings.h> | ||
| #include <filesystem> | ||
| #include <fstream> | ||
| #include <archive.h> | ||
| #include <archive_entry.h> | ||
|
|
||
| #include <switch.h> | ||
| #include <chrono> | ||
|
|
||
| #include "util/progress_event.hpp" | ||
|
|
||
| using namespace brls::literals; // for _i18n | ||
| namespace fs = std::filesystem; | ||
|
|
||
| constexpr size_t WRITE_BUFFER_SIZE = 0x10000; | ||
| using ArchivePtr = std::unique_ptr<struct archive, decltype(&archive_read_free)>; | ||
|
|
||
| namespace extract | ||
| { | ||
| namespace | ||
| std::tuple<int64_t, int64_t> getFileStats(const std::string &archivePath) | ||
| { | ||
| bool caselessCompare(const std::string &a, const std::string &b) | ||
| { | ||
| return strcasecmp(a.c_str(), b.c_str()) == 0; | ||
| std::tuple<int64_t, int64_t> stats{0, 0}; | ||
| ArchivePtr archive(archive_read_new(), archive_read_free); | ||
| struct archive_entry* entry; | ||
|
|
||
| archive_read_support_format_all(archive.get()); | ||
| archive_read_support_filter_all(archive.get()); | ||
|
|
||
| if(archive_read_open_filename(archive.get(), archivePath.c_str(), 10240) == ARCHIVE_OK) { | ||
| while(archive_read_next_header(archive.get(), &entry) == ARCHIVE_OK) { | ||
| std::get<0>(stats) += 1; | ||
| std::get<1>(stats) += archive_entry_size(entry); | ||
| } | ||
| } | ||
|
|
||
| s64 getUncompressedSize(const std::string &archivePath) | ||
| return stats; | ||
| } | ||
|
|
||
| void ensureAvailableStorage(size_t uncompressedSize) | ||
| { | ||
| s64 freeStorage; | ||
|
|
||
| if (R_SUCCEEDED(nsGetFreeSpaceSize(NcmStorageId_SdCard, &freeStorage))) | ||
| { | ||
| s64 size = 0; | ||
| unzFile zfile = unzOpen(archivePath.c_str()); | ||
| unz_global_info gi; | ||
| unzGetGlobalInfo(zfile, &gi); | ||
| for (uLong i = 0; i < gi.number_entry; ++i) | ||
| brls::Logger::info("Uncompressed size of archive: {}. Available: {}", uncompressedSize, freeStorage); | ||
| if (uncompressedSize * 1.1 > freeStorage) | ||
| { | ||
| unz_file_info fi; | ||
| unzOpenCurrentFile(zfile); | ||
| unzGetCurrentFileInfo(zfile, &fi, NULL, 0, NULL, 0, NULL, 0); | ||
| size += fi.uncompressed_size; | ||
| unzCloseCurrentFile(zfile); | ||
| unzGoToNextFile(zfile); | ||
| brls::Application::crash("app/errors/insufficient_storage"_i18n); | ||
| std::this_thread::sleep_for(std::chrono::microseconds(2000000)); | ||
| brls::Application::quit(); | ||
| } | ||
| unzClose(zfile); | ||
| return size; // in B | ||
| } | ||
| } | ||
|
|
||
| void ensureAvailableStorage(const std::string &archivePath) | ||
| { | ||
| s64 uncompressedSize = getUncompressedSize(archivePath); | ||
| s64 freeStorage; | ||
| void extract(const std::string &archivePath, const std::string &workingPath, bool overwriteExisting) { | ||
| auto start = std::chrono::high_resolution_clock::now(); | ||
| int count = 0; | ||
|
|
||
| if (R_SUCCEEDED(nsGetFreeSpaceSize(NcmStorageId_SdCard, &freeStorage))) | ||
| { | ||
| brls::Logger::info("Uncompressed size of archive {}: {}. Available: {}", archivePath, uncompressedSize, freeStorage); | ||
| if (uncompressedSize * 1.1 > freeStorage) | ||
| { | ||
| brls::Application::crash("app/errors/insufficient_storage"_i18n); | ||
| std::this_thread::sleep_for(std::chrono::microseconds(2000000)); | ||
| brls::Application::quit(); | ||
| } | ||
| } | ||
| try { | ||
|
|
||
| auto [totalFiles, totalSize] = getFileStats(archivePath); | ||
| ensureAvailableStorage(totalSize); | ||
|
|
||
| brls::sync([totalFiles, totalSize]() { | ||
| brls::Logger::info("Extracting {} entries of size {} bytes", totalFiles, totalSize); | ||
| }); | ||
|
|
||
| ProgressEvent::instance().setTotalSteps(totalFiles); | ||
| ProgressEvent::instance().setStep(0); | ||
|
|
||
| ArchivePtr archive(archive_read_new(), archive_read_free); | ||
| struct archive_entry *entry; | ||
| int err = 0, i = 0; | ||
|
|
||
| archive_read_support_format_all(archive.get()); | ||
| archive_read_support_filter_all(archive.get()); | ||
|
|
||
| if ((err = archive_read_open_filename(archive.get(), archivePath.c_str(), 10240))) { | ||
| brls::sync([err = std::string(archive_error_string(archive.get()))]() { | ||
| brls::Logger::error("Error opening archive: {}", err); | ||
| }); | ||
| return; | ||
| } | ||
|
|
||
| void extractEntry(std::string filename, unzFile &zfile, bool forceCreateTree = false) | ||
| { | ||
| if (filename.back() == '/') | ||
| { | ||
| fs::create_directories(filename); | ||
| return; | ||
| for (;;) { | ||
| if (ProgressEvent::instance().getInterupt()) { | ||
| ProgressEvent::instance().setStep(ProgressEvent::instance().getMax()); | ||
| break; | ||
| } | ||
| if (forceCreateTree) | ||
| { | ||
| fs::create_directories(filename); | ||
|
|
||
| err = archive_read_next_header(archive.get(), &entry); | ||
| if (err == ARCHIVE_EOF) { | ||
| ProgressEvent::instance().setStep(ProgressEvent::instance().getMax()); | ||
| break; | ||
| } | ||
| void *buf = malloc(WRITE_BUFFER_SIZE); | ||
| FILE *outfile; | ||
| outfile = fopen(filename.c_str(), "wb"); | ||
| for (int j = unzReadCurrentFile(zfile, buf, WRITE_BUFFER_SIZE); j > 0; j = unzReadCurrentFile(zfile, buf, WRITE_BUFFER_SIZE)) | ||
| { | ||
| fwrite(buf, 1, j, outfile); | ||
| if (err < ARCHIVE_OK) | ||
| brls::sync([archivePath, err = std::string(archive_error_string(archive.get()))]() { | ||
| brls::Logger::error("Error reading archive entry: {}", err); | ||
| }); | ||
| if (err < ARCHIVE_WARN) { | ||
| break; | ||
| } | ||
| free(buf); | ||
| fclose(outfile); | ||
| } | ||
| } | ||
|
|
||
| void extract(const std::string &archivePath, const std::string &workingPath, bool overwriteExisting, std::function<void()> func) | ||
| { | ||
| ensureAvailableStorage(archivePath); | ||
|
|
||
| unzFile zfile = unzOpen(archivePath.c_str()); | ||
| unz_global_info gi; | ||
| unzGetGlobalInfo(zfile, &gi); | ||
| auto filepath = fs::path(workingPath) / archive_entry_pathname(entry); | ||
|
|
||
| ProgressEvent::instance().setTotalSteps(gi.number_entry); | ||
| ProgressEvent::instance().setStep(0); | ||
| if (archive_entry_filetype(entry) == AE_IFDIR) { | ||
| fs::create_directories(filepath); | ||
| ProgressEvent::instance().setStep(++i); | ||
| continue; | ||
| } | ||
|
|
||
| for (uLong i = 0; i < gi.number_entry; ++i) | ||
| { | ||
| char szFilename[0x301] = ""; | ||
| unzOpenCurrentFile(zfile); | ||
| unzGetCurrentFileInfo(zfile, NULL, szFilename, sizeof(szFilename), NULL, 0, NULL, 0); | ||
| std::string filename = workingPath + szFilename; | ||
| if (fs::exists(filepath) && !overwriteExisting) { | ||
| ProgressEvent::instance().setStep(++i); | ||
| continue; | ||
| } | ||
|
|
||
| if (ProgressEvent::instance().getInterupt()) | ||
| { | ||
| unzCloseCurrentFile(zfile); | ||
| std::ofstream outfile(filepath.string(), std::ios::binary | std::ios::trunc); | ||
| if (!outfile.is_open()) { | ||
| brls::sync([path = filepath.string()]() { | ||
| brls::Logger::error("Error opening write file for archive entry: {}", path); | ||
| }); | ||
| break; | ||
| } | ||
|
|
||
| if (overwriteExisting || !fs::exists(filename)) | ||
| { | ||
| ProgressEvent::instance().setMsg(filename); | ||
| extractEntry(filename, zfile); | ||
| const void* buff = nullptr; | ||
| size_t size = 0; | ||
| int64_t offset = 0; | ||
| int res = -1; | ||
| while ((res = archive_read_data_block(archive.get(), &buff, &size, &offset)) == ARCHIVE_OK) { | ||
| try { | ||
| outfile.write(static_cast<const char*>(buff), size); | ||
| } catch(const std::exception& e) { | ||
| res = ARCHIVE_FATAL; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| if (res != ARCHIVE_EOF) { | ||
| brls::sync([res = std::string(archive_error_string(archive.get()))]() { | ||
| brls::Logger::error("Error writing out archive entry: {}", res); | ||
| }); | ||
| outfile.close(); | ||
| fs::remove(filepath); | ||
| break; | ||
| } | ||
|
|
||
| ProgressEvent::instance().setStep(i); | ||
| unzCloseCurrentFile(zfile); | ||
| unzGoToNextFile(zfile); | ||
| count++; | ||
| ProgressEvent::instance().setStep(++i); | ||
| } | ||
|
|
||
| } catch(const std::exception& e) { | ||
| brls::sync([e = std::string(e.what())]() { | ||
| brls::Logger::error("Unexpected error extracting archive: {}", e); | ||
| }); | ||
| } | ||
| unzClose(zfile); | ||
| ProgressEvent::instance().setStep(ProgressEvent::instance().getMax()); | ||
|
|
||
| auto end = std::chrono::high_resolution_clock::now(); | ||
| auto elapsed = std::chrono::duration_cast<std::chrono::seconds>(end - start).count(); | ||
|
|
||
| brls::sync([elapsed, count]() { | ||
| brls::Logger::info("Total extraction time: {}s for {} files", elapsed, count); | ||
| }); | ||
| } | ||
| } |
Oops, something went wrong.