Merge pull request #19 from jprzimba/server-accepts-utf8

[FEATURE]  Using `boost/locale` to convert text to UTF-8.

cmake/modules/BaseConfig.cmake

@@ -27,6 +27,7 @@ find_package(absl CONFIG REQUIRED)
 find_package(asio CONFIG REQUIRED)
 find_package(eventpp CONFIG REQUIRED)
 find_package(magic_enum CONFIG REQUIRED)
+find_package(Boost REQUIRED COMPONENTS locale)
 if(FEATURE_METRICS)
     find_package(opentelemetry-cpp CONFIG REQUIRED)
     find_package(prometheus-cpp CONFIG REQUIRED)

cmake/modules/CrystalLib.cmake

@@ -86,6 +86,7 @@ target_link_libraries(${PROJECT_NAME}_lib
         unofficial::argon2::libargon2
         unofficial::libmariadb
         protobuf
+        Boost::locale
 )
 
 if(FEATURE_METRICS)

src/io/functions/iologindata_save_player.cpp

@@ -818,9 +818,10 @@ bool IOLoginDataSave::savePlayerStatement(const std::shared_ptr<Player> &player,
 	Database &db = Database::getInstance();
 	std::ostringstream query;
 
+	std::string utf8Text = convertToUTF8(text);
 	query << "INSERT INTO `player_statements` (`player_id`, `receiver`, `channel_id`, `text`, `date`) VALUES ("
 		  << player->getGUID() << ", " << db.escapeString(receiver) << ", " << channelId << ", "
-		  << db.escapeString(text) << ", " << time(nullptr) << ")";
+		  << db.escapeString(utf8Text) << ", " << time(nullptr) << ")";
 
 	if (!db.executeQuery(query.str())) {
 		return false;

src/server/network/message/networkmessage.cpp

@@ -16,8 +16,8 @@
 ////////////////////////////////////////////////////////////////////////
 
 #include "server/network/message/networkmessage.hpp"
-
 #include "items/containers/container.hpp"
+#include <boost/locale.hpp>
 
 int32_t NetworkMessage::decodeHeader() {
 	// Ensure there are enough bytes to read the header (2 bytes)
@@ -112,9 +112,12 @@ std::string NetworkMessage::getString(uint16_t stringLen /* = 0*/, const std::so
 	g_logger().trace("[{}] called line '{}:{}' in '{}'", __FUNCTION__, location.line(), location.column(), location.function_name());
 
 	// Copy the string from the buffer
-	std::string result(buffer.begin() + info.position, buffer.begin() + info.position + stringLen);
+	auto it = buffer.data() + info.position;
 	info.position += stringLen;
-	return result;
+
+	// Convert the string to UTF-8 using Boost.Locale
+	std::string_view latin1Str { reinterpret_cast<const char*>(it), stringLen };
+	return boost::locale::conv::to_utf<char>(latin1Str.data(), latin1Str.data() + latin1Str.size(), "ISO-8859-1", boost::locale::conv::skip);
 }
 
 Position NetworkMessage::getPosition() {
@@ -131,18 +134,28 @@ void NetworkMessage::skipBytes(int16_t count) {
 }
 
 void NetworkMessage::addString(const std::string &value, const std::source_location &location /*= std::source_location::current()*/, const std::string &function /* = ""*/) {
-	size_t stringLen = value.length();
 	if (value.empty()) {
 		if (!function.empty()) {
 			g_logger().debug("[{}] attempted to add an empty string. Called line '{}'", __FUNCTION__, function);
 		} else {
 			g_logger().debug("[{}] attempted to add an empty string. Called line '{}:{}' in '{}'", __FUNCTION__, location.line(), location.column(), location.function_name());
 		}
 
-		// Add a 0 length string, the std::array will be filled with 0s
-		add<uint16_t>(uint16_t());
+		// Add a 0 length string
+		add<uint16_t>(0);
 		return;
 	}
+
+	// Convert to ISO-8859-1 using Boost.Locale
+	std::string latin1Str = boost::locale::conv::from_utf<char>(
+		value.data(),
+		value.data() + value.size(),
+		"ISO-8859-1",
+		boost::locale::conv::skip
+	);
+
+	size_t stringLen = latin1Str.size();
+
 	if (!canAdd(stringLen + 2)) {
 		if (!function.empty()) {
 			g_logger().error("[{}] NetworkMessage size is wrong: {}. Called line '{}'", __FUNCTION__, stringLen, function);
@@ -151,6 +164,7 @@ void NetworkMessage::addString(const std::string &value, const std::source_locat
 		}
 		return;
 	}
+
 	if (stringLen > NETWORKMESSAGE_MAXSIZE) {
 		if (!function.empty()) {
 			g_logger().error("[{}] exceeded NetworkMessage max size: {}, actual size: {}. Called line '{}'", __FUNCTION__, NETWORKMESSAGE_MAXSIZE, stringLen, function);
@@ -166,10 +180,12 @@ void NetworkMessage::addString(const std::string &value, const std::source_locat
 		g_logger().trace("[{}] called line '{}:{}' in '{}'", __FUNCTION__, location.line(), location.column(), location.function_name());
 	}
 
-	auto len = static_cast<uint16_t>(stringLen);
-	add<uint16_t>(len);
-	// Using to copy the string into the buffer
-	std::ranges::copy(value, buffer.begin() + info.position);
+	// Add the string length to the buffer
+	add<uint16_t>(static_cast<uint16_t>(stringLen));
+
+	// Copy the Latin-1 encoded string to the buffer
+	std::memcpy(buffer.data() + info.position, latin1Str.data(), stringLen);
+
 	info.position += stringLen;
 	info.length += stringLen;
 }

src/utils/tools.cpp

@@ -27,6 +27,8 @@
 #include "absl/debugging/stacktrace.h"
 #include "absl/debugging/symbolize.h"
 
+#include <boost/locale.hpp>
+
 void printXMLError(const std::string &where, const std::string &fileName, const pugi::xml_parse_result &result) {
 	g_logger().error("[{}] Failed to load {}: {}", where, fileName, result.description());
 
@@ -2134,3 +2136,7 @@ uint8_t calculateMaxPvpReduction(uint8_t blessCount, bool isPromoted /* = false*
 
 	return result;
 }
+
+std::string convertToUTF8(const std::string &input) {
+	return boost::locale::conv::to_utf<char>(input, "ISO-8859-1");
+}

src/utils/tools.hpp

@@ -223,3 +223,5 @@ const std::map<uint8_t, uint16_t> &getMaxValuePerSkill();
 
 float calculateEquipmentLoss(uint8_t blessingAmount, bool isContainer = false);
 uint8_t calculateMaxPvpReduction(uint8_t blessCount, bool isPromoted = false);
+
+std::string convertToUTF8(const std::string &input);

vcpkg.json

@@ -20,6 +20,7 @@
     "zlib",
     "bshoshany-thread-pool",
     "atomic-queue",
+    "boost-locale",
     {
       "name": "opentelemetry-cpp",
       "default-features": true,