Support Unicode and long-paths on Windows

Makefile.am

@@ -46,7 +46,8 @@ libpar2_a_SOURCES = src/crc.cpp src/crc.h \
 	src/reedsolomon.cpp src/reedsolomon.h \
 	src/verificationhashtable.cpp src/verificationhashtable.h \
 	src/verificationpacket.cpp src/verificationpacket.h \
-	src/libpar2.cpp src/libpar2.h src/libpar2internal.h
+	src/libpar2.cpp src/libpar2.h src/libpar2internal.h \
+	src/utf8.cpp src/utf8.h
 
 
 bin_PROGRAMS = par2
@@ -117,6 +118,7 @@ tests_crc_test_SOURCES = src/crc_test.cpp src/crc.cpp src/crc.h
 tests_md5_test_SOURCES = src/md5_test.cpp src/md5.cpp src/md5.h
 
 tests_diskfile_test_SOURCES = src/diskfile_test.cpp src/diskfile.cpp src/diskfile.h
+tests_diskfile_test_LDADD = libpar2.a
 
 tests_libpar2_test_SOURCES = src/libpar2_test.cpp src/libpar2.h
 tests_libpar2_test_LDADD = libpar2.a

par2cmdline.vcxproj

@@ -31,17 +31,17 @@
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
     <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
     <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
     <PlatformToolset>v141_xp</PlatformToolset>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
@@ -200,6 +200,7 @@
     <ClCompile Include="src\par2repairersourcefile.cpp" />
     <ClCompile Include="src\recoverypacket.cpp" />
     <ClCompile Include="src\reedsolomon.cpp" />
+    <ClCompile Include="src\utf8.cpp" />
     <ClCompile Include="src\verificationhashtable.cpp" />
     <ClCompile Include="src\verificationpacket.cpp" />
   </ItemGroup>
@@ -228,6 +229,7 @@
     <ClInclude Include="src\par2repairersourcefile.h" />
     <ClInclude Include="src\recoverypacket.h" />
     <ClInclude Include="src\reedsolomon.h" />
+    <ClInclude Include="src\utf8.h" />
     <ClInclude Include="src\verificationhashtable.h" />
     <ClInclude Include="src\verificationpacket.h" />
   </ItemGroup>
@@ -255,4 +257,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>

par2cmdline.vcxproj.filters

@@ -90,6 +90,9 @@
     <ClCompile Include="src\libpar2.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="src\utf8.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="src\commandline.h">
@@ -170,6 +173,9 @@
     <ClInclude Include="src\libpar2internal.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="src\utf8.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="AUTHORS" />

src/commandline.cpp

@@ -1320,10 +1320,17 @@ bool CommandLine::ComputeRecoveryBlockCount(u32 *recoveryblockcount,
 bool CommandLine::SetParFilename(string filename)
 {
   bool result = false;
+  string::size_type position = 0;
   string::size_type where;
 
-  if ((where = filename.find_first_of('*')) != string::npos ||
-      (where = filename.find_first_of('?')) != string::npos)
+#ifdef _WIN32
+  if (filename.rfind(R"(\\?\)", 0) == 0) {
+    position = strlen(R"(\\?\)");
+  }
+#endif
+
+  if ((where = filename.find_first_of('*', position)) != string::npos ||
+    (where = filename.find_first_of('?', position)) != string::npos)
   {
     cerr << "par2 file must not have a wildcard in it." << endl;
     return result;

src/diskfile.cpp

@@ -69,14 +69,14 @@ bool DiskFile::CreateParentDirectory(string _pathname)
   {
     string path = filename.substr(0, where);
 
-    struct stat st;
-    if (stat(path.c_str(), &st) == 0)
+    struct struct_stat st;
+    if (stat(utf8::widen(path).c_str(), &st) == 0)
       return true; // let the caller deal with non-directories
 
     if (!DiskFile::CreateParentDirectory(path))
       return false;
 
-    if (!CreateDirectory(path.c_str(), NULL))
+    if (!CreateDirectory(utf8::widen(path).c_str(), NULL))
     {
       DWORD error = ::GetLastError();
 
@@ -101,7 +101,7 @@ bool DiskFile::Create(string _filename, u64 _filesize)
     return false;
 
   // Create the file
-  hFile = ::CreateFileA(_filename.c_str(), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_NEW, 0, NULL);
+  hFile = ::CreateFile(utf8::widen(_filename).c_str(), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_NEW, 0, NULL);
   if (hFile == INVALID_HANDLE_VALUE)
   {
     DWORD error = ::GetLastError();
@@ -126,7 +126,7 @@ bool DiskFile::Create(string _filename, u64 _filesize)
 
       ::CloseHandle(hFile);
       hFile = INVALID_HANDLE_VALUE;
-      ::DeleteFile(_filename.c_str());
+      ::DeleteFile(utf8::widen(_filename).c_str());
 
       return false;
     }
@@ -140,7 +140,7 @@ bool DiskFile::Create(string _filename, u64 _filesize)
 
       ::CloseHandle(hFile);
       hFile = INVALID_HANDLE_VALUE;
-      ::DeleteFile(_filename.c_str());
+      ::DeleteFile(utf8::widen(_filename).c_str());
 
       return false;
     }
@@ -223,7 +223,7 @@ bool DiskFile::Open(const string &_filename, u64 _filesize)
   filename = _filename;
   filesize = _filesize;
 
-  hFile = ::CreateFileA(_filename.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
+  hFile = ::CreateFile(utf8::widen(_filename).c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
   if (hFile == INVALID_HANDLE_VALUE)
   {
     DWORD error = ::GetLastError();
@@ -315,43 +315,54 @@ void DiskFile::Close(void)
 
 string DiskFile::GetCanonicalPathname(string filename)
 {
-  char fullname[MAX_PATH];
-  char *filepart;
+#ifdef UNICODE
+	typedef wstring TSTRING;
+#else
+	typedef string TSTRING;
+#endif
+
+  if (filename.rfind(R"(\\?\)", 0) == 0)
+  {
+    return filename;
+  }
+
+  TCHAR fullname[MAX_PATH];
+  TCHAR *filepart;
 
   // Resolve a relative path to a full path
-  unsigned int length = ::GetFullPathName(filename.c_str(), sizeof(fullname), fullname, &filepart);
-  if (length <= 0 || sizeof(fullname) < length)
+  unsigned int length = ::GetFullPathName(utf8::widen(filename).c_str(), sizeof(fullname)/sizeof(TCHAR), fullname, &filepart);
+  if (length <= 0 || (sizeof(fullname)/sizeof(TCHAR)) < length)
     return filename;
 
   // Make sure the drive letter is upper case.
   fullname[0] = toupper(fullname[0]);
 
   // Translate all /'s to \'s
-  char *current = strchr(fullname, '/');
+  TCHAR *current = _tcschr(fullname, '/');
   while (current)
   {
     *current++ = '\\';
-    current  = strchr(current, '/');
+    current  = _tcschr(current, '/');
   }
 
   // Copy the root directory to the output string
-  string longname(fullname, 3);
+  TSTRING longname(fullname, 3);
 
   // Start processing at the first path component
   current = &fullname[3];
-  char *limit = &fullname[length];
+  TCHAR *limit = &fullname[length];
 
   // Process until we reach the end of the full name
   while (current < limit)
   {
-    char *tail;
+    TCHAR *tail;
 
     // Find the next \, or the end of the string
-    (tail = strchr(current, '\\')) || (tail = limit);
+    (tail = _tcschr(current, '\\')) || (tail = limit);
     *tail = 0;
 
     // Create a wildcard to search for the path
-    string wild = longname + current;
+    TSTRING wild = longname + current;
     WIN32_FIND_DATA finddata;
     HANDLE hFind = ::FindFirstFile(wild.c_str(), &finddata);
     if (hFind == INVALID_HANDLE_VALUE)
@@ -373,7 +384,7 @@ string DiskFile::GetCanonicalPathname(string filename)
       longname += '\\';
   }
 
-  return longname;
+  return utf8::narrow(longname.c_str());
 }
 
 std::unique_ptr< list<string> > DiskFile::FindFiles(string path, string wildcard, bool recursive)
@@ -388,14 +399,14 @@ std::unique_ptr< list<string> > DiskFile::FindFiles(string path, string wildcard
 
   wildcard = path + wildcard;
   WIN32_FIND_DATA fd;
-  HANDLE h = ::FindFirstFile(wildcard.c_str(), &fd);
+  HANDLE h = ::FindFirstFile(utf8::widen(wildcard).c_str(), &fd);
   if (h != INVALID_HANDLE_VALUE)
   {
     do
     {
       if (0 == (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
       {
-        matches->push_back(path + fd.cFileName);
+        matches->push_back(path + utf8::narrow(fd.cFileName));
       }
       else if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
       {
@@ -405,7 +416,7 @@ std::unique_ptr< list<string> > DiskFile::FindFiles(string path, string wildcard
 
         string nwwildcard="*";
 	std::unique_ptr< list<string> > dirmatches(
-						 DiskFile::FindFiles(path + fd.cFileName, nwwildcard, true)
+						 DiskFile::FindFiles(path + utf8::narrow(fd.cFileName), nwwildcard, true)
 						 );
 
         matches->merge(*dirmatches);
@@ -419,8 +430,8 @@ std::unique_ptr< list<string> > DiskFile::FindFiles(string path, string wildcard
 
 u64 DiskFile::GetFileSize(string filename)
 {
-  struct _stati64 st;
-  if ((0 == _stati64(filename.c_str(), &st)) && (0 != (st.st_mode & S_IFREG)))
+  struct struct_stat st;
+  if ((0 == stat(utf8::widen(filename).c_str(), &st)) && (0 != (st.st_mode & S_IFREG)))
   {
     return st.st_size;
   }
@@ -432,8 +443,8 @@ u64 DiskFile::GetFileSize(string filename)
 
 bool DiskFile::FileExists(string filename)
 {
-  struct _stati64 st;
-  return ((0 == _stati64(filename.c_str(), &st)) && (0 != (st.st_mode & S_IFREG)));
+  struct struct_stat st;
+  return ((0 == stat(utf8::widen(filename).c_str(), &st)) && (0 != (st.st_mode & S_IFREG)));
 }
 
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1000,7 +1011,11 @@ bool DiskFile::Rename(void)
   char newname[_MAX_PATH+1];
   u32 index = 0;
 
+#ifdef _WIN32
+  struct struct_stat st;
+#else
   struct stat st;
+#endif
 
   do
   {
@@ -1016,7 +1031,7 @@ bool DiskFile::Rename(void)
       return false;
     }
     newname[length] = 0;
-  } while (stat(newname, &st) == 0);
+  } while (stat(utf8::widen(newname).c_str(), &st) == 0);
 
   return Rename(newname);
 }
@@ -1029,7 +1044,7 @@ bool DiskFile::Rename(string _filename)
   assert(file == 0);
 #endif
 
-  if (::rename(filename.c_str(), _filename.c_str()) == 0)
+  if (::rename(utf8::widen(filename).c_str(), utf8::widen(_filename).c_str()) == 0)
   {
     filename = _filename;
 

src/libpar2internal.h

@@ -36,12 +36,21 @@
 #include <io.h>
 #include <fcntl.h>
 #include <assert.h>
+#include <tchar.h>
 
 #define snprintf _snprintf_s
 #define sprintf  sprintf_s
 #define stricmp  _stricmp
 #define unlink   _unlink
-#define stat _stat
+
+#ifdef UNICODE
+#define stat _wstati64
+#define struct_stat _stati64
+#define rename _wrename
+#else
+#define stat _stati64
+#define struct_stat _stati64
+#endif
 
 #define __LITTLE_ENDIAN 1234
 #define __BIG_ENDIAN    4321
@@ -194,6 +203,7 @@ char *strchr(), *strrchr();
 #define LONGMULTIPLY
 
 // STL includes
+#include <string>
 #include <list>
 #include <map>
 #include <algorithm>
@@ -245,6 +255,8 @@ using namespace std;
 #include "par1repairersourcefile.h"
 #include "par1repairer.h"
 
+#include "utf8.h"
+
 // Heap checking
 #ifdef _MSC_VER
 #define _CRTDBG_MAP_ALLOC

src/par2cmdline.cpp

@@ -32,11 +32,40 @@ static char THIS_FILE[]=__FILE__;
 #endif
 #endif
 
-int main(int argc, char *argv[])
+#if defined(_MSC_VER)
+extern int __cdecl realmain(int argc, char* argv[]);
+
+int wmain(int argc, wchar_t* argv[])
+{
+  char** args;
+  int bytesneeded = sizeof(char*) * (argc + 1);
+  HANDLE heap = GetProcessHeap();
+
+  for (int i = 0; i < argc; ++i)
+    bytesneeded += WideCharToMultiByte(CP_UTF8, 0, argv[i], -1, NULL, 0, NULL, NULL);
+
+  args = static_cast<char**>(HeapAlloc(heap, 0, bytesneeded));
+  args[0] = (char*)(args + argc + 1);
+
+  for (int i = 0; i < argc; ++i)
+    args[i + 1] = args[i] + WideCharToMultiByte(CP_UTF8, 0, argv[i], -1, args[i], bytesneeded, NULL, NULL);
+
+  args[argc] = NULL;
+
+  argc = realmain(argc, args);
+  HeapFree(heap, 0, args);
+  return argc;
+}
+
+int realmain(int argc, char* argv[])
+#else 
+int main(int argc, char* argv[])
+#endif
 {
 #ifdef _MSC_VER
   // Memory leak checking
   _CrtSetDbgFlag(_CrtSetDbgFlag(_CRTDBG_REPORT_FLAG) | _CRTDBG_ALLOC_MEM_DF | /*_CRTDBG_CHECK_CRT_DF | */_CRTDBG_DELAY_FREE_MEM_DF | _CRTDBG_LEAK_CHECK_DF);
+  SetConsoleOutputCP(CP_UTF8);
 #endif
 
   // check sizeof integers