From 771248fcf4e021ba6e1fca9afd39f119de7bb6ca Mon Sep 17 00:00:00 2001
From: kaetemi <kaetemi@gmail.com>
Date: Sat, 24 Oct 2020 04:18:25 +0800
Subject: [PATCH] Use platform implementation in ucstring for UTF-8 conversion
 on Windows to get valid UTF-16

---
 nel/include/nel/misc/stream.h   |  7 +++++++
 nel/include/nel/misc/types_nl.h |  4 ++++
 nel/include/nel/misc/ucstring.h |  4 ++++
 nel/src/misc/seven_zip.cpp      |  4 ++--
 nel/src/misc/string_common.cpp  |  2 +-
 nel/src/misc/ucstring.cpp       | 17 +++++++++++++++++
 6 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/nel/include/nel/misc/stream.h b/nel/include/nel/misc/stream.h
index 89db78a1c..35b6c3f26 100644
--- a/nel/include/nel/misc/stream.h
+++ b/nel/include/nel/misc/stream.h
@@ -300,6 +300,13 @@ public:
 	virtual void	serial(bool &b) ;
 #ifndef NL_OS_CYGWIN
 	virtual void	serial(char &b) ;
+#endif
+#ifdef NL_OS_WINDOWS
+	inline void     serial(wchar_t &b)
+	{ 
+		nlctassert(sizeof(wchar_t) == sizeof(uint16)); 
+		serial(reinterpret_cast<uint16 &>(b));
+	}
 #endif
 	virtual void	serial(std::string &b) ;
 	virtual void	serial(ucstring &b) ;
diff --git a/nel/include/nel/misc/types_nl.h b/nel/include/nel/misc/types_nl.h
index 445882c34..73c05b3a5 100644
--- a/nel/include/nel/misc/types_nl.h
+++ b/nel/include/nel/misc/types_nl.h
@@ -546,7 +546,11 @@ template<> struct hash<uint64>
  * \typedef ucchar
  * An Unicode character (16 bits)
  */
+#if defined(NL_OS_WINDOWS)
+typedef	wchar_t ucchar;
+#else
 typedef	uint16	ucchar;
+#endif
 
 #ifndef NL_OVERRIDE
 #define NL_OVERRIDE override
diff --git a/nel/include/nel/misc/ucstring.h b/nel/include/nel/misc/ucstring.h
index 28afcdb9a..8859a21d9 100644
--- a/nel/include/nel/misc/ucstring.h
+++ b/nel/include/nel/misc/ucstring.h
@@ -30,7 +30,11 @@
  * An unicode string class (16 bits per character).
  * Add features to convert and assign \c ucstring to \c string and \c string to \c ucstring.
  */
+#if defined(NL_OS_WINDOWS)
+typedef std::wstring ucstringbase;
+#else
 typedef std::basic_string<ucchar> ucstringbase;
+#endif
 
 class ucstring : public ucstringbase
 {
diff --git a/nel/src/misc/seven_zip.cpp b/nel/src/misc/seven_zip.cpp
index ff226af7d..ad07bf3cc 100644
--- a/nel/src/misc/seven_zip.cpp
+++ b/nel/src/misc/seven_zip.cpp
@@ -172,8 +172,8 @@ bool unpack7Zip(const std::string &sevenZipFile, const std::string &destFileName
 	filename.resize(nameLen);
 
 	// write filename into ucstring
-	SzArEx_GetFileNameUtf16(&db, 0, &filename[0]);
-
+	SzArEx_GetFileNameUtf16(&db, 0, reinterpret_cast<UInt16 *>(&filename[0]));
+	
 	// write the extracted file
 	FILE *outputHandle = nlfopen(destFileName, "wb+");
 
diff --git a/nel/src/misc/string_common.cpp b/nel/src/misc/string_common.cpp
index a0fb40942..93122cc70 100644
--- a/nel/src/misc/string_common.cpp
+++ b/nel/src/misc/string_common.cpp
@@ -244,7 +244,7 @@ std::wstring utf8ToWide(const char *str, size_t len)
 #if defined(NL_OS_WINDOWS)
 	return winCpToWide(str, len, CP_UTF8);
 #else
-	// TODO: UTF-32 to UTF-8
+	// TODO: UTF-8 to UTF-32
 	nlassert(false);
 #endif
 }
diff --git a/nel/src/misc/ucstring.cpp b/nel/src/misc/ucstring.cpp
index 8f81cbea1..5fad643ce 100644
--- a/nel/src/misc/ucstring.cpp
+++ b/nel/src/misc/ucstring.cpp
@@ -31,6 +31,12 @@ void ucstring::toString(std::string &str) const
 
 std::string ucstring::toUtf8() const
 {
+#if defined(NL_OS_WINDOWS)
+	// Use OS implementation
+	nlctassert(sizeof(wchar_t) == sizeof(ucchar));
+	nlctassert(sizeof(wchar_t) == sizeof(uint16));
+	return NLMISC::wideToUtf8(static_cast<const std::wstring &>(*this));
+#else
 	std::string res;
 	ucstring::const_iterator first(begin()), last(end());
 	for (; first != last; ++first)
@@ -65,10 +71,20 @@ std::string ucstring::toUtf8() const
 		}
 	}
 	return res;
+#endif
 }
 
 void ucstring::fromUtf8(const std::string &stringUtf8)
 {
+#if defined(NL_OS_WINDOWS)
+	// Use OS implementation
+	nlctassert(sizeof(wchar_t) == sizeof(ucchar));
+	nlctassert(sizeof(wchar_t) == sizeof(uint16));
+	nlctassert(sizeof(std::wstring) == sizeof(ucstring)); // These can be swapped on Windows
+	static_cast<std::wstring &>(*this) = nlmove(NLMISC::utf8ToWide(stringUtf8));
+	if (stringUtf8.size() && !size())
+		rawCopy(stringUtf8);
+#else
 	// clear the string
 	erase();
 
@@ -146,6 +162,7 @@ void ucstring::fromUtf8(const std::string &stringUtf8)
 			push_back(code);
 		}
 	}
+#endif
 }
 
 void ucstring::rawCopy(const std::string &str)