Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Windows utf-16 Paths #424

Merged
merged 1 commit into from
Jan 10, 2025
Merged

feat: Windows utf-16 Paths #424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 51 additions & 1 deletion src/filesystem/api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
Expand All @@ -25,11 +25,19 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once

#include <codecvt>
#include <locale>
#include <string>

#include "../status.h"
#include "google/protobuf/message.h"

#ifdef _WIN32
// suppress the min and max definitions in Windef.h.
#define NOMINMAX
#include <Windows.h>
#endif

namespace triton { namespace core {

enum class FileSystemType { LOCAL, GCS, S3, AS };
Expand Down Expand Up @@ -68,6 +76,48 @@ class LocalizedPath {
// FIXME: Remove when no longer required
std::vector<std::shared_ptr<LocalizedPath>> other_localized_path;

#ifdef _WIN32
//! Converts incoming utf-8 path to an OS valid path
//!
//! On Linux there is not much to do but make sure correct slashes are used
//! On Windows we need to take care of the long paths and handle them
//! correctly to avoid legacy issues with MAX_PATH
//!
//! More details:
//! https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry
//!
static inline std::wstring GetWindowsValidPath(const std::string& path)
{
std::string l_path(path);
// On Windows long paths must be marked correctly otherwise, due to
// backwards compatibility, all paths are limited to MAX_PATH length
static constexpr const char* kWindowsLongPathPrefix = "\\\\?\\";
if (l_path.size() >= MAX_PATH) {
// Must be prefixed with "\\?\" to be considered long path
if (l_path.substr(0, 4) != (kWindowsLongPathPrefix)) {
// Long path but not "tagged" correctly
l_path = (kWindowsLongPathPrefix) + l_path;
}
}
std::replace(l_path.begin(), l_path.end(), '/', '\\');
return String2Wstring(l_path);
}
#endif // _WIN32

static inline std::wstring String2Wstring(const std::string& str)
{
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::wstring wide = converter.from_bytes(str);
return wide;
}

static inline std::string Wstring2String(const std::wstring& str)
{
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
std::string narrow = converter.to_bytes(str);
return narrow;
}

private:
std::string original_path_;
std::string local_path_;
Expand Down
139 changes: 79 additions & 60 deletions src/filesystem/implementations/local.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -60,44 +60,18 @@ class LocalFileSystem : public FileSystem {
Status MakeTemporaryDirectory(
std::string dir_path, std::string* temp_dir) override;
Status DeletePath(const std::string& path) override;

private:
inline std::string GetOSValidPath(const std::string& path);
};

//! Converts incoming utf-8 path to an OS valid path
//!
//! On Linux there is not much to do but make sure correct slashes are used
//! On Windows we need to take care of the long paths and handle them correctly
//! to avoid legacy issues with MAX_PATH
//!
//! More details:
//! https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry
//!
inline std::string
LocalFileSystem::GetOSValidPath(const std::string& path)
{
std::string l_path(path);
#ifdef _WIN32
// On Windows long paths must be marked correctly otherwise, due to backwards
// compatibility, all paths are limited to MAX_PATH length
static constexpr const char* kWindowsLongPathPrefix = "\\\\?\\";
if (l_path.size() >= MAX_PATH) {
// Must be prefixed with "\\?\" to be considered long path
if (l_path.substr(0, 4) != (kWindowsLongPathPrefix)) {
// Long path but not "tagged" correctly
l_path = (kWindowsLongPathPrefix) + l_path;
}
}
std::replace(l_path.begin(), l_path.end(), '/', '\\');
#endif
return l_path;
}

Status
LocalFileSystem::FileExists(const std::string& path, bool* exists)
{
*exists = (access(GetOSValidPath(path).c_str(), F_OK) == 0);
#ifdef _WIN32
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);
*exists = (_waccess(wpath.c_str(), F_OK) == 0);
#else
*exists = (access(path.c_str(), F_OK) == 0);
#endif // _WIN32

return Status::Success;
}

Expand All @@ -106,11 +80,18 @@ LocalFileSystem::IsDirectory(const std::string& path, bool* is_dir)
{
*is_dir = false;

#ifdef _WIN32
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);
struct _stat64i32 st;
if (_wstat64i32(wpath.c_str(), &st) != 0) {
return Status(Status::Code::INTERNAL, "failed to stat file " + path);
}
#else
struct stat st;
if (stat(GetOSValidPath(path).c_str(), &st) != 0) {
if (stat(path.c_str(), &st) != 0) {
return Status(Status::Code::INTERNAL, "failed to stat file " + path);
}

#endif // _WIN32
*is_dir = S_ISDIR(st.st_mode);
return Status::Success;
}
Expand All @@ -119,15 +100,20 @@ Status
LocalFileSystem::FileModificationTime(
const std::string& path, int64_t* mtime_ns)
{
struct stat st;
if (stat(GetOSValidPath(path).c_str(), &st) != 0) {
#ifdef _WIN32
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);

struct _stat64i32 st;
if (_wstat64i32(wpath.c_str(), &st) != 0) {
return Status(Status::Code::INTERNAL, "failed to stat file " + path);
}

#ifdef _WIN32
// In Windows, st_mtime is in time_t
*mtime_ns = std::max(st.st_mtime, st.st_ctime);
#else
struct stat st;
if (stat(path.c_str(), &st) != 0) {
return Status(Status::Code::INTERNAL, "failed to stat file " + path);
}
*mtime_ns =
std::max(TIMESPEC_TO_NANOS(st.st_mtim), TIMESPEC_TO_NANOS(st.st_ctim));
#endif
Expand All @@ -139,20 +125,23 @@ LocalFileSystem::GetDirectoryContents(
const std::string& path, std::set<std::string>* contents)
{
#ifdef _WIN32
WIN32_FIND_DATA entry;
WIN32_FIND_DATAW entry;
// Append "*" to obtain all files under 'path'
HANDLE dir = FindFirstFile(JoinPath({path, "*"}).c_str(), &entry);
std::string jointPath = JoinPath({path, "*"});
std::wstring wpath = LocalizedPath::GetWindowsValidPath(jointPath);

HANDLE dir = FindFirstFileW(wpath.c_str(), &entry);
if (dir == INVALID_HANDLE_VALUE) {
return Status(Status::Code::INTERNAL, "failed to open directory " + path);
}
if ((strcmp(entry.cFileName, ".") != 0) &&
(strcmp(entry.cFileName, "..") != 0)) {
contents->insert(entry.cFileName);
if ((wcscmp(entry.cFileName, L".") != 0) &&
(wcscmp(entry.cFileName, L"..") != 0)) {
contents->insert(LocalizedPath::Wstring2String(entry.cFileName));
}
while (FindNextFile(dir, &entry)) {
if ((strcmp(entry.cFileName, ".") != 0) &&
(strcmp(entry.cFileName, "..") != 0)) {
contents->insert(entry.cFileName);
while (FindNextFileW(dir, &entry)) {
if ((wcscmp(entry.cFileName, L".") != 0) &&
(wcscmp(entry.cFileName, L"..") != 0)) {
contents->insert(LocalizedPath::Wstring2String(entry.cFileName));
}
}

Expand Down Expand Up @@ -219,7 +208,13 @@ LocalFileSystem::GetDirectoryFiles(
Status
LocalFileSystem::ReadTextFile(const std::string& path, std::string* contents)
{
std::ifstream in(GetOSValidPath(path), std::ios::in | std::ios::binary);
#ifdef _WIN32
std::wstring valid_path = LocalizedPath::GetWindowsValidPath(path);
#else
std::string valid_path(path);
#endif

std::ifstream in(valid_path, std::ios::in | std::ios::binary);
if (!in) {
return Status(
Status::Code::INTERNAL,
Expand Down Expand Up @@ -249,6 +244,12 @@ Status
LocalFileSystem::WriteTextFile(
const std::string& path, const std::string& contents)
{
#ifdef _WIN32
std::wstring valid_path = LocalizedPath::GetWindowsValidPath(path);
#else
std::string valid_path(path);
#endif

std::ofstream out(path, std::ios::out | std::ios::binary);
if (!out) {
return Status(
Expand All @@ -266,7 +267,13 @@ Status
LocalFileSystem::WriteBinaryFile(
const std::string& path, const char* contents, const size_t content_len)
{
std::ofstream out(path, std::ios::out | std::ios::binary);
#ifdef _WIN32
std::wstring valid_path = LocalizedPath::GetWindowsValidPath(path);
#else
std::string valid_path(path);
#endif

std::ofstream out(valid_path, std::ios::out | std::ios::binary);
if (!out) {
return Status(
Status::Code::INTERNAL, "failed to open binary file for write " + path +
Expand All @@ -282,7 +289,8 @@ Status
LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive)
{
#ifdef _WIN32
if (mkdir(dir.c_str()) == -1)
std::wstring wdir = LocalizedPath::GetWindowsValidPath(dir);
if (_wmkdir(wdir.c_str()) == -1)
#else
if (mkdir(dir.c_str(), S_IRWXU) == -1)
#endif
Expand All @@ -293,7 +301,7 @@ LocalFileSystem::MakeDirectory(const std::string& dir, const bool recursive)
RETURN_IF_ERROR(MakeDirectory(DirName(dir), recursive));
// Retry the creation
#ifdef _WIN32
if (mkdir(dir.c_str()) == -1)
if (_wmkdir(wdir.c_str()) == -1)
#else
if (mkdir(dir.c_str(), S_IRWXU) == -1)
#endif
Expand All @@ -317,8 +325,8 @@ LocalFileSystem::MakeTemporaryDirectory(
std::string dir_path, std::string* temp_dir)
{
#ifdef _WIN32
char temp_path[MAX_PATH + 1];
size_t temp_path_length = GetTempPath(MAX_PATH + 1, temp_path);
wchar_t temp_path[MAX_PATH + 1];
size_t temp_path_length = GetTempPathW(MAX_PATH + 1, temp_path);
if (temp_path_length == 0) {
return Status(
Status::Code::INTERNAL,
Expand All @@ -336,13 +344,14 @@ LocalFileSystem::MakeTemporaryDirectory(
std::lock_guard<std::mutex> lk(mtx);
// Construct a std::string as filled 'temp_path' is not C string,
// and so that we can reuse 'temp_path' to hold the temp file name.
std::string temp_path_str(temp_path, temp_path_length);
if (GetTempFileName(temp_path_str.c_str(), "folder", 0, temp_path) == 0) {
std::wstring temp_path_str(temp_path, temp_path_length);
if (GetTempFileNameW(temp_path_str.c_str(), L"folder", 0, temp_path) == 0) {
return Status(Status::Code::INTERNAL, "Failed to create local temp folder");
}
*temp_dir = temp_path;
DeleteFile(temp_dir->c_str());
if (CreateDirectory(temp_dir->c_str(), NULL) == 0) {
std::wstring wstr_temp_path = temp_path;
*temp_dir = LocalizedPath::Wstring2String(wstr_temp_path);
DeleteFileW(wstr_temp_path.c_str());
if (CreateDirectoryW(wstr_temp_path.c_str(), NULL) == 0) {
return Status(
Status::Code::INTERNAL,
"Failed to create local temp folder: " + *temp_dir);
Expand Down Expand Up @@ -375,9 +384,19 @@ LocalFileSystem::DeletePath(const std::string& path)
for (const auto& content : contents) {
RETURN_IF_ERROR(DeletePath(JoinPath({path, content})));
}
#ifdef _WIN32
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);
_wrmdir(wpath.c_str());
#else
rmdir(path.c_str());
#endif // _WIN32
} else {
#ifdef _WIN32
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);
_wremove(wpath.c_str());
#else
remove(path.c_str());
#endif // _WIN32
}
return Status::Success;
}
Expand Down
8 changes: 5 additions & 3 deletions src/shared_library.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ SharedLibrary::SetLibraryDirectory(const std::string& path)
{
#ifdef _WIN32
LOG_VERBOSE(1) << "SetLibraryDirectory: path = " << path;
if (!SetDllDirectory(path.c_str())) {
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);
if (!SetDllDirectoryW(wpath.c_str())) {
LPSTR err_buffer = nullptr;
size_t size = FormatMessageA(
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
Expand All @@ -88,7 +89,7 @@ SharedLibrary::ResetLibraryDirectory()
{
#ifdef _WIN32
LOG_VERBOSE(1) << "ResetLibraryDirectory";
if (!SetDllDirectory(NULL)) {
if (!SetDllDirectoryW(NULL)) {
LPSTR err_buffer = nullptr;
size_t size = FormatMessageA(
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
Expand Down Expand Up @@ -130,7 +131,8 @@ SharedLibrary::OpenLibraryHandle(const std::string& path, void** handle)
// HMODULE is typedef of void*
// https://docs.microsoft.com/en-us/windows/win32/winprog/windows-data-types
LOG_VERBOSE(1) << "OpenLibraryHandle: path = " << path;
*handle = LoadLibrary(path.c_str());
std::wstring wpath = LocalizedPath::GetWindowsValidPath(path);
*handle = LoadLibraryW(wpath.c_str());

// Remove the dll path added above... do this unconditionally before
// check for failure in dll load.
Expand Down
Loading